1 /* 2 * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shared/barrierSet.hpp" 27 #include "gc/shared/barrierSetAssembler.hpp" 28 #include "gc/shared/barrierSetNMethod.hpp" 29 #include "gc/shared/collectedHeap.hpp" 30 #include "interpreter/interp_masm.hpp" 31 #include "memory/universe.hpp" 32 #include "runtime/jniHandles.hpp" 33 #include "runtime/sharedRuntime.hpp" 34 #include "runtime/thread.hpp" 35 36 #define __ masm-> 37 38 void BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 39 Register dst, Address src, Register tmp1, Register tmp_thread) { 40 bool in_heap = (decorators & IN_HEAP) != 0; 41 bool in_native = (decorators & IN_NATIVE) != 0; 42 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 43 bool atomic = (decorators & MO_RELAXED) != 0; 44 45 switch (type) { 46 case T_OBJECT: 47 case T_ARRAY: { 48 if (in_heap) { 49 #ifdef _LP64 50 if (UseCompressedOops) { 51 __ movl(dst, src); 52 if (is_not_null) { 53 __ decode_heap_oop_not_null(dst); 54 } else { 55 __ decode_heap_oop(dst); 56 } 57 } else 58 #endif 59 { 60 __ movptr(dst, src); 61 } 62 } else { 63 assert(in_native, "why else?"); 64 __ movptr(dst, src); 65 } 66 break; 67 } 68 case T_BOOLEAN: __ load_unsigned_byte(dst, src); break; 69 case T_BYTE: __ load_signed_byte(dst, src); break; 70 case T_CHAR: __ load_unsigned_short(dst, src); break; 71 case T_SHORT: __ load_signed_short(dst, src); break; 72 case T_INT: __ movl (dst, src); break; 73 case T_ADDRESS: __ movptr(dst, src); break; 74 case T_FLOAT: 75 assert(dst == noreg, "only to ftos"); 76 __ load_float(src); 77 break; 78 case T_DOUBLE: 79 assert(dst == noreg, "only to dtos"); 80 __ load_double(src); 81 break; 82 case T_LONG: 83 assert(dst == noreg, "only to ltos"); 84 #ifdef _LP64 85 __ movq(rax, src); 86 #else 87 if (atomic) { 88 __ fild_d(src); // Must load atomically 89 __ subptr(rsp,2*wordSize); // Make space for store 90 __ fistp_d(Address(rsp,0)); 91 __ pop(rax); 92 __ pop(rdx); 93 } else { 94 __ movl(rax, src); 95 __ movl(rdx, src.plus_disp(wordSize)); 96 } 97 #endif 98 break; 99 default: Unimplemented(); 100 } 101 } 102 103 void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 104 Address dst, Register val, Register tmp1, Register tmp2) { 105 bool in_heap = (decorators & IN_HEAP) != 0; 106 bool in_native = (decorators & IN_NATIVE) != 0; 107 bool is_not_null = (decorators & IS_NOT_NULL) != 0; 108 bool atomic = (decorators & MO_RELAXED) != 0; 109 110 switch (type) { 111 case T_OBJECT: 112 case T_ARRAY: { 113 if (in_heap) { 114 if (val == noreg) { 115 assert(!is_not_null, "inconsistent access"); 116 #ifdef _LP64 117 if (UseCompressedOops) { 118 __ movl(dst, (int32_t)NULL_WORD); 119 } else { 120 __ movslq(dst, (int32_t)NULL_WORD); 121 } 122 #else 123 __ movl(dst, (int32_t)NULL_WORD); 124 #endif 125 } else { 126 #ifdef _LP64 127 if (UseCompressedOops) { 128 assert(!dst.uses(val), "not enough registers"); 129 if (is_not_null) { 130 __ encode_heap_oop_not_null(val); 131 } else { 132 __ encode_heap_oop(val); 133 } 134 __ movl(dst, val); 135 } else 136 #endif 137 { 138 __ movptr(dst, val); 139 } 140 } 141 } else { 142 assert(in_native, "why else?"); 143 assert(val != noreg, "not supported"); 144 __ movptr(dst, val); 145 } 146 break; 147 } 148 case T_BOOLEAN: 149 __ andl(val, 0x1); // boolean is true if LSB is 1 150 __ movb(dst, val); 151 break; 152 case T_BYTE: 153 __ movb(dst, val); 154 break; 155 case T_SHORT: 156 __ movw(dst, val); 157 break; 158 case T_CHAR: 159 __ movw(dst, val); 160 break; 161 case T_INT: 162 __ movl(dst, val); 163 break; 164 case T_LONG: 165 assert(val == noreg, "only tos"); 166 #ifdef _LP64 167 __ movq(dst, rax); 168 #else 169 if (atomic) { 170 __ push(rdx); 171 __ push(rax); // Must update atomically with FIST 172 __ fild_d(Address(rsp,0)); // So load into FPU register 173 __ fistp_d(dst); // and put into memory atomically 174 __ addptr(rsp, 2*wordSize); 175 } else { 176 __ movptr(dst, rax); 177 __ movptr(dst.plus_disp(wordSize), rdx); 178 } 179 #endif 180 break; 181 case T_FLOAT: 182 assert(val == noreg, "only tos"); 183 __ store_float(dst); 184 break; 185 case T_DOUBLE: 186 assert(val == noreg, "only tos"); 187 __ store_double(dst); 188 break; 189 case T_ADDRESS: 190 __ movptr(dst, val); 191 break; 192 default: Unimplemented(); 193 } 194 } 195 196 #ifndef _LP64 197 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 198 Address obj1, jobject obj2) { 199 __ cmpoop_raw(obj1, obj2); 200 } 201 202 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 203 Register obj1, jobject obj2) { 204 __ cmpoop_raw(obj1, obj2); 205 } 206 #endif 207 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 208 Register obj1, Address obj2) { 209 __ cmpptr(obj1, obj2); 210 } 211 212 void BarrierSetAssembler::obj_equals(MacroAssembler* masm, 213 Register obj1, Register obj2) { 214 __ cmpptr(obj1, obj2); 215 } 216 217 void BarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, 218 Register obj, Register tmp, Label& slowpath) { 219 __ clear_jweak_tag(obj); 220 __ movptr(obj, Address(obj, 0)); 221 } 222 223 void BarrierSetAssembler::tlab_allocate(MacroAssembler* masm, 224 Register thread, Register obj, 225 Register var_size_in_bytes, 226 int con_size_in_bytes, 227 Register t1, 228 Register t2, 229 Label& slow_case) { 230 assert_different_registers(obj, t1, t2); 231 assert_different_registers(obj, var_size_in_bytes, t1); 232 Register end = t2; 233 if (!thread->is_valid()) { 234 #ifdef _LP64 235 thread = r15_thread; 236 #else 237 assert(t1->is_valid(), "need temp reg"); 238 thread = t1; 239 __ get_thread(thread); 240 #endif 241 } 242 243 __ verify_tlab(); 244 245 __ movptr(obj, Address(thread, JavaThread::tlab_top_offset())); 246 if (var_size_in_bytes == noreg) { 247 __ lea(end, Address(obj, con_size_in_bytes)); 248 } else { 249 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 250 } 251 __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset())); 252 __ jcc(Assembler::above, slow_case); 253 254 // update the tlab top pointer 255 __ movptr(Address(thread, JavaThread::tlab_top_offset()), end); 256 257 // recover var_size_in_bytes if necessary 258 if (var_size_in_bytes == end) { 259 __ subptr(var_size_in_bytes, obj); 260 } 261 __ verify_tlab(); 262 } 263 264 // Defines obj, preserves var_size_in_bytes 265 void BarrierSetAssembler::eden_allocate(MacroAssembler* masm, 266 Register thread, Register obj, 267 Register var_size_in_bytes, 268 int con_size_in_bytes, 269 Register t1, 270 Label& slow_case) { 271 assert(obj == rax, "obj must be in rax, for cmpxchg"); 272 assert_different_registers(obj, var_size_in_bytes, t1); 273 if (!Universe::heap()->supports_inline_contig_alloc()) { 274 __ jmp(slow_case); 275 } else { 276 Register end = t1; 277 Label retry; 278 __ bind(retry); 279 ExternalAddress heap_top((address) Universe::heap()->top_addr()); 280 __ movptr(obj, heap_top); 281 if (var_size_in_bytes == noreg) { 282 __ lea(end, Address(obj, con_size_in_bytes)); 283 } else { 284 __ lea(end, Address(obj, var_size_in_bytes, Address::times_1)); 285 } 286 // if end < obj then we wrapped around => object too long => slow case 287 __ cmpptr(end, obj); 288 __ jcc(Assembler::below, slow_case); 289 __ cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr())); 290 __ jcc(Assembler::above, slow_case); 291 // Compare obj with the top addr, and if still equal, store the new top addr in 292 // end at the address of the top addr pointer. Sets ZF if was equal, and clears 293 // it otherwise. Use lock prefix for atomicity on MPs. 294 __ locked_cmpxchgptr(end, heap_top); 295 __ jcc(Assembler::notEqual, retry); 296 incr_allocated_bytes(masm, thread, var_size_in_bytes, con_size_in_bytes, thread->is_valid() ? noreg : t1); 297 } 298 } 299 300 void BarrierSetAssembler::incr_allocated_bytes(MacroAssembler* masm, Register thread, 301 Register var_size_in_bytes, 302 int con_size_in_bytes, 303 Register t1) { 304 if (!thread->is_valid()) { 305 #ifdef _LP64 306 thread = r15_thread; 307 #else 308 assert(t1->is_valid(), "need temp reg"); 309 thread = t1; 310 __ get_thread(thread); 311 #endif 312 } 313 314 #ifdef _LP64 315 if (var_size_in_bytes->is_valid()) { 316 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 317 } else { 318 __ addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 319 } 320 #else 321 if (var_size_in_bytes->is_valid()) { 322 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes); 323 } else { 324 __ addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes); 325 } 326 __ adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0); 327 #endif 328 } 329 330 #ifdef _LP64 331 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { 332 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 333 if (bs_nm == NULL) { 334 return; 335 } 336 Label continuation; 337 Register thread = r15_thread; 338 Address disarmed_addr(thread, in_bytes(bs_nm->thread_disarmed_offset())); 339 __ align(8); 340 __ cmpl(disarmed_addr, 0); 341 __ jcc(Assembler::equal, continuation); 342 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); 343 __ bind(continuation); 344 } 345 #else 346 void BarrierSetAssembler::nmethod_entry_barrier(MacroAssembler* masm) { 347 BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod(); 348 if (bs_nm == NULL) { 349 return; 350 } 351 352 Label continuation; 353 354 Register tmp = rdi; 355 __ push(tmp); 356 __ movptr(tmp, (intptr_t)bs_nm->disarmed_value_address()); 357 Address disarmed_addr(tmp, 0); 358 __ align(4); 359 __ cmpl(disarmed_addr, 0); 360 __ pop(tmp); 361 __ jcc(Assembler::equal, continuation); 362 __ call(RuntimeAddress(StubRoutines::x86::method_entry_barrier())); 363 __ bind(continuation); 364 } 365 #endif 366 367 void BarrierSetAssembler::c2i_entry_barrier(MacroAssembler* masm) { 368 BarrierSetNMethod* bs = BarrierSet::barrier_set()->barrier_set_nmethod(); 369 if (bs == NULL) { 370 return; 371 } 372 373 Label bad_call; 374 __ cmpptr(rbx, 0); // rbx contains the incoming method for c2i adapters. 375 __ jcc(Assembler::equal, bad_call); 376 377 // Pointer chase to the method holder to find out if the method is concurrently unloading. 378 Label method_live; 379 __ load_method_holder_cld(rscratch1, rbx); 380 381 // Is it a strong CLD? 382 __ movl(rscratch2, Address(rscratch1, ClassLoaderData::keep_alive_offset())); 383 __ cmpptr(rscratch2, 0); 384 __ jcc(Assembler::greater, method_live); 385 386 // Is it a weak but alive CLD? 387 __ movptr(rscratch1, Address(rscratch1, ClassLoaderData::holder_offset())); 388 __ resolve_weak_handle(rscratch1, rscratch2); 389 __ cmpptr(rscratch1, 0); 390 __ jcc(Assembler::notEqual, method_live); 391 392 __ bind(bad_call); 393 __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 394 __ bind(method_live); 395 }