1 /* 2 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "gc/shared/barrierSet.hpp" 28 #include "gc/shared/barrierSetAssembler.hpp" 29 #include "gc/shared/barrierSetNMethod.hpp" 30 #include "gc/shared/barrierSetRuntime.hpp" 31 #include "gc/shared/collectedHeap.hpp" 32 #include "interpreter/interp_masm.hpp" 33 #include "memory/universe.hpp" 34 #include "runtime/jniHandles.hpp" 35 #include "runtime/sharedRuntime.hpp" 36 #include "runtime/thread.hpp" 37 38 #define __ masm-> 39 40 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 41 Register dst, Address src, Register tmp1, Register tmp_thread) { 42 bool in_heap = (decorators & IN_HEAP) != 0; 43 bool in_native = (decorators & IN_NATIVE) != 0; 44 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 45 bool atomic = (decorators & MO_RELAXED) != 0; 46 47 assert(type != T_VALUETYPE, "Not supported yet"); 48 switch (type) { 49 case T_OBJECT: 50 case T_ARRAY: { 51 if (in_heap) { 52 #ifdef _LP64 53 if (UseCompressedOops) { 54 __ movl(dst, src); 55 if (is_not_null) { 56 __ decode_heap_oop_not_null(dst); 57 } else { 58 __ decode_heap_oop(dst); 59 } 60 } else 61 #endif 62 { 63 __ movptr(dst, src); 64 } 65 } else { 66 assert(in_native, "why else?"); 67 __ movptr(dst, src); 68 } 69 break; 70 } 71 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break; 72 case T_BYTE: __ load_signed_byte(dst, src); break; 73 case T_CHAR: __ load_unsigned_short(dst, src); break; 74 case T_SHORT: __ load_signed_short(dst, src); break; 75 case T_INT: __ movl (dst, src); break; 76 case T_ADDRESS: __ movptr(dst, src); break; 77 case T_FLOAT: 78 assert(dst == noreg, "only to ftos"); 79 __ load_float(src); 80 break; 81 case T_DOUBLE: 82 assert(dst == noreg, "only to dtos"); 83 __ load_double(src); 84 break; 85 case T_LONG: 86 assert(dst == noreg, "only to ltos"); 87 #ifdef _LP64 88 __ movq(rax, src); 89 #else 90 if (atomic) { 91 __ fild_d(src); // Must load atomically 92 __ subptr(rsp,2*wordSize); // Make space for store 93 __ fistp_d(Address(rsp,0)); 94 __ pop(rax); 95 __ pop(rdx); 96 } else { 97 __ movl(rax, src); 98 __ movl(rdx, src.plus_disp(wordSize)); 99 } 100 #endif 101 break; 102 default: Unimplemented(); 103 } 104 } 105 106 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 107 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 108 bool in_heap = (decorators & IN_HEAP) != 0; 109 bool in_native = (decorators & IN_NATIVE) != 0; 110 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 111 bool atomic = (decorators & MO_RELAXED) != 0; 112 113 assert(type != T_VALUETYPE, "Not supported yet"); 114 switch (type) { 115 case T_OBJECT: 116 case T_ARRAY: { 117 if (in_heap) { 118 if (val == noreg) { 119 assert(!is_not_null, "inconsistent access"); 120 #ifdef _LP64 121 if (UseCompressedOops) { 122 __ movl(dst, (int32_t)NULL_WORD); 123 } else { 124 __ movslq(dst, (int32_t)NULL_WORD); 125 } 126 #else 127 __ movl(dst, (int32_t)NULL_WORD); 128 #endif 129 } else { 130 #ifdef _LP64 131 if (UseCompressedOops) { 132 assert(!dst.uses(val), "not enough registers"); 133 if (is_not_null) { 134 __ encode_heap_oop_not_null(val); 135 } else { 136 __ encode_heap_oop(val); 137 } 138 __ movl(dst, val); 139 } else 140 #endif 141 { 142 __ movptr(dst, val); 143 } 144 } 145 } else { 146 assert(in_native, "why else?"); 147 assert(val != noreg, "not supported"); 148 __ movptr(dst, val); 149 } 150 break; 151 } 152 case T_BOOLEAN: 153 __ andl(val, 0x1); // boolean is true if LSB is 1 154 __ movb(dst, val); 155 break; 156 case T_BYTE: 157 __ movb(dst, val); 158 break; 159 case T_SHORT: 160 __ movw(dst, val); 161 break; 162 case T_CHAR: 163 __ movw(dst, val); 164 break; 165 case T_INT: 166 __ movl(dst, val); 167 break; 168 case T_LONG: 169 assert(val == noreg, "only tos"); 170 #ifdef _LP64 171 __ movq(dst, rax); 172 #else 173 if (atomic) { 174 __ push(rdx); 175 __ push(rax); // Must update atomically with FIST 176 __ fild_d(Address(rsp,0)); // So load into FPU register 177 __ fistp_d(dst); // and put into memory atomically 178 __ addptr(rsp, 2*wordSize); 179 } else { 180 __ movptr(dst, rax); 181 __ movptr(dst.plus_disp(wordSize), rdx); 182 } 183 #endif 184 break; 185 case T_FLOAT: 186 assert(val == noreg, "only tos"); 187 __ store_float(dst); 188 break; 189 case T_DOUBLE: 190 assert(val == noreg, "only tos"); 191 __ store_double(dst); 192 break; 193 case T_ADDRESS: 194 __ movptr(dst, val); 195 break; 196 default: Unimplemented(); 197 } 198 } 199 200 void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators, 201 Register src, Register dst, Register value_klass) { 202 // value_copy implementation is fairly complex, and there are not any 203 // "short-cuts" to be made from asm. What there is, appears to have the same 204 // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds 205 // of hand-rolled instructions... 206 if (decorators & IS_DEST_UNINITIALIZED) { 207 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass); 208 } else { 209 __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass); 210 } 211 } 212 213 214 #ifndef _LP64 215 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 216 Address obj1, jobject obj2) { 217 __ cmpoop_raw(obj1, obj2); 218 } 219 220 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 221 Register obj1, jobject obj2) { 222 __ cmpoop_raw(obj1, obj2); 223 } 224 #endif 225 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 226 Register obj1, Address obj2) { 227 __ cmpptr(obj1, obj2); 228 } 229 230 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 231 Register obj1, Register obj2) { 232 __ cmpptr(obj1, obj2); 233 } 234 235 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 236 Register obj, Register tmp, Label& slowpath) { 237 __ clear_jweak_tag(obj); 238 __ movptr(obj, Address(obj, 0)); 239 } 240 241 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, 242 Register thread, Register obj, 243 Register var_size_in_bytes, 244 int con_size_in_bytes, 245 Register t1, 246 Register t2, 247 Label& slow_case) { 248 assert_different_registers(obj, t1, t2); 249 assert_different_registers(obj, var_size_in_bytes, t1); 250 Register end = t2; 251 if (!thread->is_valid()) { 252 #ifdef _LP64 253 thread = r15_thread; 254 #else 255 assert(t1->is_valid(), "need temp reg"); 256 thread = t1; 257 __ get_thread(thread); 258 #endif 259 } 260 261 __ verify_tlab(); 262 263 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); 264 if (var_size_in_bytes == noreg) { 265 __ lea(end, Address(obj, con_size_in_bytes)); 266 } else { 267 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 268 } 269 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); 270 __ jcc(Assembler::above, slow_case); 271 272 // update the tlab top pointer 273 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); 274 275 // recover var_size_in_bytes if necessary 276 if (var_size_in_bytes == end) { 277 __ subptr(var_size_in_bytes, obj); 278 } 279 __ verify_tlab(); 280 } 281 282 // Defines obj, preserves var_size_in_bytes 283 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, 284 Register thread, Register obj, 285 Register var_size_in_bytes, 286 int con_size_in_bytes, 287 Register t1, 288 Label& slow_case) { 289 assert(obj == rax, "obj must be in rax, for cmpxchg"); 290 assert_different_registers(obj, var_size_in_bytes, t1); 291 if (!Universe::heap()->supports_inline_contig_alloc()) { 292 __ jmp(slow_case); 293 } else { 294 Register end = t1; 295 Label retry; 296 __ bind(retry); 297 ExternalAddress heap_top((address) Universe::heap()->top_addr()); 298 __ movptr(obj, heap_top); 299 if (var_size_in_bytes == noreg) { 300 __ lea(end, Address(obj, con_size_in_bytes)); 301 } else { 302 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 303 } 304 // if end < obj then we wrapped around => object too long => slow case 305 __ cmpptr(end, obj); 306 __ jcc(Assembler::below, slow_case); 307 __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr())); 308 __ jcc(Assembler::above, slow_case); 309 // Compare obj with the top addr, and if still equal, store the new top addr in 310 // end at the address of the top addr pointer. Sets ZF if was equal, and clears 311 // it otherwise. Use lock prefix for atomicity on MPs. 312 __ locked_cmpxchgptr(end, heap_top); 313 __ jcc(Assembler::notEqual, retry); 314 incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1); 315 } 316 } 317 318 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread, 319 Register var_size_in_bytes, 320 int con_size_in_bytes, 321 Register t1) { 322 if (!thread->is_valid()) { 323 #ifdef _LP64 324 thread = r15_thread; 325 #else 326 assert(t1->is_valid(), "need temp reg"); 327 thread = t1; 328 __ get_thread(thread); 329 #endif 330 } 331 332 #ifdef _LP64 333 if (var_size_in_bytes->is_valid()) { 334 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 335 } else { 336 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 337 } 338 #else 339 if (var_size_in_bytes->is_valid()) { 340 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 341 } else { 342 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 343 } 344 __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); 345 #endif 346 } 347 348 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { 349 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 350 if (bs_nm == NULL) { 351 return; 352 } 353 #ifndef _LP64 354 ShouldNotReachHere(); 355 #else 356 Label continuation; 357 Register thread = LP64_ONLY(r15_thread); 358 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset())); 359 __ align(8); 360 __ cmpl(disarmed_addr, 0); 361 __ jcc(Assembler::equal, continuation); 362 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); 363 __ bind(continuation); 364 #endif 365 } 366 367 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { 368 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod(); 369 if (bs == NULL) { 370 return; 371 } 372 373 Label bad_call; 374 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters. 375 __ jcc(Assembler::equal, bad_call); 376 377 // Pointer chase to the method holder to find out if the method is concurrently unloading. 378 Label method_live; 379 __ load_method_holder_cld(rscratch1, rbx); 380 381 // Is it a strong CLD? 382 __ movl(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset())); 383 __ cmpptr(rscratch2, 0); 384 __ jcc(Assembler::greater, method_live); 385 386 // Is it a weak but alive CLD? 387 __ movptr(rscratch1, Address(rscratch1, ClassLoaderData::holder_offset())); 388 __ resolve_weak_handle(rscratch1, rscratch2); 389 __ cmpptr(rscratch1, 0); 390 __ jcc(Assembler::notEqual, method_live); 391 392 __ bind(bad_call); 393 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 394 __ bind(method_live); 395 }