1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "c1/c1_LIRAssembler.hpp" 28 #include "c1/c1_MacroAssembler.hpp" 29 #include "gc/g1/c1/g1BarrierSetC1.hpp" 30 #include "gc/g1/g1BarrierSet.hpp" 31 #include "gc/g1/g1BarrierSetAssembler.hpp" 32 #include "gc/g1/g1BarrierSetRuntime.hpp" 33 #include "gc/g1/g1CardTable.hpp" 34 #include "gc/g1/g1ThreadLocalData.hpp" 35 #include "gc/g1/heapRegion.hpp" 36 #include "gc/shared/collectedHeap.hpp" 37 #include "runtime/sharedRuntime.hpp" 38 #include "runtime/thread.hpp" 39 #include "interpreter/interp_masm.hpp" 40 #include "runtime/sharedRuntime.hpp" 41 42 #define __ masm-> 43 44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 45 Register addr, Register count, RegSet saved_regs) { 46 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0; 47 if (!dest_uninitialized) { 48 __ push(saved_regs, sp); 49 if (count == c_rarg0) { 50 if (addr == c_rarg1) { 51 // exactly backwards!! 52 __ mov(rscratch1, c_rarg0); 53 __ mov(c_rarg0, c_rarg1); 54 __ mov(c_rarg1, rscratch1); 55 } else { 56 __ mov(c_rarg1, count); 57 __ mov(c_rarg0, addr); 58 } 59 } else { 60 __ mov(c_rarg0, addr); 61 __ mov(c_rarg1, count); 62 } 63 if (UseCompressedOops) { 64 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_narrow_oop_entry), 2); 65 } else { 66 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_pre_oop_entry), 2); 67 } 68 __ pop(saved_regs, sp); 69 } 70 } 71 72 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 73 Register start, Register end, Register scratch, RegSet saved_regs) { 74 __ push(saved_regs, sp); 75 // must compute element count unless barrier set interface is changed (other platforms supply count) 76 assert_different_registers(start, end, scratch); 77 __ lea(scratch, Address(end, BytesPerHeapOop)); 78 __ sub(scratch, scratch, start); // subtract start to get #bytes 79 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count 80 __ mov(c_rarg0, start); 81 __ mov(c_rarg1, scratch); 82 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_array_post_entry), 2); 83 __ pop(saved_regs, sp); 84 } 85 86 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, 87 Register obj, 88 Register pre_val, 89 Register thread, 90 Register tmp, 91 bool tosca_live, 92 bool expand_call) { 93 // If expand_call is true then we expand the call_VM_leaf macro 94 // directly to skip generating the check by 95 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 96 97 assert(thread == rthread, "must be"); 98 99 Label done; 100 Label runtime; 101 102 assert_different_registers(obj, pre_val, tmp, rscratch1); 103 assert(pre_val != noreg && tmp != noreg, "expecting a register"); 104 105 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 106 Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 107 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 108 109 // Is marking active? 110 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 111 __ ldrw(tmp, in_progress); 112 } else { 113 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 114 __ ldrb(tmp, in_progress); 115 } 116 __ cbzw(tmp, done); 117 118 // Do we need to load the previous value? 119 if (obj != noreg) { 120 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW); 121 } 122 123 // Is the previous value null? 124 __ cbz(pre_val, done); 125 126 // Can we store original value in the thread's buffer? 127 // Is index == 0? 128 // (The index field is typed as size_t.) 129 130 __ ldr(tmp, index); // tmp := *index_adr 131 __ cbz(tmp, runtime); // tmp == 0? 132 // If yes, goto runtime 133 134 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize 135 __ str(tmp, index); // *index_adr := tmp 136 __ ldr(rscratch1, buffer); 137 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr 138 139 // Record the previous value 140 __ str(pre_val, Address(tmp, 0)); 141 __ b(done); 142 143 __ bind(runtime); 144 // save the live input values 145 RegSet saved = RegSet::of(pre_val); 146 if (tosca_live) saved += RegSet::of(r0); 147 if (obj != noreg) saved += RegSet::of(obj); 148 149 __ push(saved, sp); 150 151 // Calling the runtime using the regular call_VM_leaf mechanism generates 152 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 153 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. 154 // 155 // If we care generating the pre-barrier without a frame (e.g. in the 156 // intrinsified Reference.get() routine) then ebp might be pointing to 157 // the caller frame and so this check will most likely fail at runtime. 158 // 159 // Expanding the call directly bypasses the generation of the check. 160 // So when we do not have have a full interpreter frame on the stack 161 // expand_call should be passed true. 162 163 if (expand_call) { 164 assert(pre_val != c_rarg1, "smashed arg"); 165 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 166 } else { 167 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 168 } 169 170 __ pop(saved, sp); 171 172 __ bind(done); 173 174 } 175 176 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, 177 Register store_addr, 178 Register new_val, 179 Register thread, 180 Register tmp, 181 Register tmp2) { 182 assert(thread == rthread, "must be"); 183 assert_different_registers(store_addr, new_val, thread, tmp, tmp2, 184 rscratch1); 185 assert(store_addr != noreg && new_val != noreg && tmp != noreg 186 && tmp2 != noreg, "expecting a register"); 187 188 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 189 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 190 191 BarrierSet* bs = BarrierSet::barrier_set(); 192 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 193 CardTable* ct = ctbs->card_table(); 194 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 195 196 Label done; 197 Label runtime; 198 199 // Does store cross heap regions? 200 201 __ eor(tmp, store_addr, new_val); 202 __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes); 203 __ cbz(tmp, done); 204 205 // crosses regions, storing NULL? 206 207 __ cbz(new_val, done); 208 209 // storing region crossing non-NULL, is card already dirty? 210 211 ExternalAddress cardtable((address) ct->byte_map_base()); 212 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 213 const Register card_addr = tmp; 214 215 __ lsr(card_addr, store_addr, CardTable::card_shift); 216 217 // get the address of the card 218 __ load_byte_map_base(tmp2); 219 __ add(card_addr, card_addr, tmp2); 220 __ ldrb(tmp2, Address(card_addr)); 221 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); 222 __ br(Assembler::EQ, done); 223 224 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 225 226 __ membar(Assembler::StoreLoad); 227 228 __ ldrb(tmp2, Address(card_addr)); 229 __ cbzw(tmp2, done); 230 231 // storing a region crossing, non-NULL oop, card is clean. 232 // dirty card and log. 233 234 __ strb(zr, Address(card_addr)); 235 236 __ ldr(rscratch1, queue_index); 237 __ cbz(rscratch1, runtime); 238 __ sub(rscratch1, rscratch1, wordSize); 239 __ str(rscratch1, queue_index); 240 241 __ ldr(tmp2, buffer); 242 __ str(card_addr, Address(tmp2, rscratch1)); 243 __ b(done); 244 245 __ bind(runtime); 246 // save the live input values 247 RegSet saved = RegSet::of(store_addr, new_val); 248 __ push(saved, sp); 249 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); 250 __ pop(saved, sp); 251 252 __ bind(done); 253 } 254 255 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 256 Register dst, Address src, Register tmp1, Register tmp_thread) { 257 bool on_oop = type == T_OBJECT || type == T_ARRAY; 258 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; 259 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; 260 bool on_reference = on_weak || on_phantom; 261 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 262 if (on_oop && on_reference) { 263 // LR is live. It must be saved around calls. 264 __ enter(); // barrier may call runtime 265 // Generate the G1 pre-barrier code to log the value of 266 // the referent field in an SATB buffer. 267 g1_write_barrier_pre(masm /* masm */, 268 noreg /* obj */, 269 dst /* pre_val */, 270 rthread /* thread */, 271 tmp1 /* tmp */, 272 true /* tosca_live */, 273 true /* expand_call */); 274 __ leave(); 275 } 276 } 277 278 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 279 Address dst, Register val, Register tmp1, Register tmp2) { 280 // flatten object address if needed 281 if (dst.index() == noreg && dst.offset() == 0) { 282 if (dst.base() != r3) { 283 __ mov(r3, dst.base()); 284 } 285 } else { 286 __ lea(r3, dst); 287 } 288 289 g1_write_barrier_pre(masm, 290 r3 /* obj */, 291 tmp2 /* pre_val */, 292 rthread /* thread */, 293 tmp1 /* tmp */, 294 val != noreg /* tosca_live */, 295 false /* expand_call */); 296 297 if (val == noreg) { 298 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg); 299 } else { 300 // G1 barrier needs uncompressed oop for region cross check. 301 Register new_val = val; 302 if (UseCompressedOops) { 303 new_val = rscratch2; 304 __ mov(new_val, val); 305 } 306 BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg); 307 g1_write_barrier_post(masm, 308 r3 /* store_adr */, 309 new_val /* new_val */, 310 rthread /* thread */, 311 tmp1 /* tmp */, 312 tmp2 /* tmp2 */); 313 } 314 315 } 316 317 #ifdef COMPILER1 318 319 #undef __ 320 #define __ ce->masm()-> 321 322 void G1BarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 323 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 324 // At this point we know that marking is in progress. 325 // If do_load() is true then we have to emit the 326 // load of the previous value; otherwise it has already 327 // been loaded into _pre_val. 328 329 __ bind(*stub->entry()); 330 331 assert(stub->pre_val()->is_register(), "Precondition."); 332 333 Register pre_val_reg = stub->pre_val()->as_register(); 334 335 if (stub->do_load()) { 336 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); 337 } 338 __ cbz(pre_val_reg, *stub->continuation()); 339 ce->store_parameter(stub->pre_val()->as_register(), 0); 340 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 341 __ b(*stub->continuation()); 342 } 343 344 void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 345 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 346 __ bind(*stub->entry()); 347 assert(stub->addr()->is_register(), "Precondition."); 348 assert(stub->new_val()->is_register(), "Precondition."); 349 Register new_val_reg = stub->new_val()->as_register(); 350 __ cbz(new_val_reg, *stub->continuation()); 351 ce->store_parameter(stub->addr()->as_pointer_register(), 0); 352 __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); 353 __ b(*stub->continuation()); 354 } 355 356 #undef __ 357 358 #define __ sasm-> 359 360 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 361 __ prologue("g1_pre_barrier", false); 362 363 // arg0 : previous value of memory 364 365 BarrierSet* bs = BarrierSet::barrier_set(); 366 367 const Register pre_val = r0; 368 const Register thread = rthread; 369 const Register tmp = rscratch1; 370 371 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 372 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 373 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 374 375 Label done; 376 Label runtime; 377 378 // Is marking still active? 379 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 380 __ ldrw(tmp, in_progress); 381 } else { 382 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 383 __ ldrb(tmp, in_progress); 384 } 385 __ cbzw(tmp, done); 386 387 // Can we store original value in the thread's buffer? 388 __ ldr(tmp, queue_index); 389 __ cbz(tmp, runtime); 390 391 __ sub(tmp, tmp, wordSize); 392 __ str(tmp, queue_index); 393 __ ldr(rscratch2, buffer); 394 __ add(tmp, tmp, rscratch2); 395 __ load_parameter(0, rscratch2); 396 __ str(rscratch2, Address(tmp, 0)); 397 __ b(done); 398 399 __ bind(runtime); 400 __ push_call_clobbered_registers(); 401 __ load_parameter(0, pre_val); 402 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_pre_entry), pre_val, thread); 403 __ pop_call_clobbered_registers(); 404 __ bind(done); 405 406 __ epilogue(); 407 } 408 409 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 410 __ prologue("g1_post_barrier", false); 411 412 // arg0: store_address 413 Address store_addr(rfp, 2*BytesPerWord); 414 415 BarrierSet* bs = BarrierSet::barrier_set(); 416 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 417 CardTable* ct = ctbs->card_table(); 418 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 419 420 Label done; 421 Label runtime; 422 423 // At this point we know new_value is non-NULL and the new_value crosses regions. 424 // Must check to see if card is already dirty 425 426 const Register thread = rthread; 427 428 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 429 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 430 431 const Register card_offset = rscratch2; 432 // LR is free here, so we can use it to hold the byte_map_base. 433 const Register byte_map_base = lr; 434 435 assert_different_registers(card_offset, byte_map_base, rscratch1); 436 437 __ load_parameter(0, card_offset); 438 __ lsr(card_offset, card_offset, CardTable::card_shift); 439 __ load_byte_map_base(byte_map_base); 440 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 441 __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val()); 442 __ br(Assembler::EQ, done); 443 444 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 445 446 __ membar(Assembler::StoreLoad); 447 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 448 __ cbzw(rscratch1, done); 449 450 // storing region crossing non-NULL, card is clean. 451 // dirty card and log. 452 __ strb(zr, Address(byte_map_base, card_offset)); 453 454 // Convert card offset into an address in card_addr 455 Register card_addr = card_offset; 456 __ add(card_addr, byte_map_base, card_addr); 457 458 __ ldr(rscratch1, queue_index); 459 __ cbz(rscratch1, runtime); 460 __ sub(rscratch1, rscratch1, wordSize); 461 __ str(rscratch1, queue_index); 462 463 // Reuse LR to hold buffer_addr 464 const Register buffer_addr = lr; 465 466 __ ldr(buffer_addr, buffer); 467 __ str(card_addr, Address(buffer_addr, rscratch1)); 468 __ b(done); 469 470 __ bind(runtime); 471 __ push_call_clobbered_registers(); 472 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSetRuntime::write_ref_field_post_entry), card_addr, thread); 473 __ pop_call_clobbered_registers(); 474 __ bind(done); 475 __ epilogue(); 476 } 477 478 #undef __ 479 480 #endif // COMPILER1