1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "c1/c1_LIRAssembler.hpp" 28 #include "c1/c1_MacroAssembler.hpp" 29 #include "gc/g1/c1/g1BarrierSetC1.hpp" 30 #include "gc/g1/g1BarrierSet.hpp" 31 #include "gc/g1/g1BarrierSetAssembler.hpp" 32 #include "gc/g1/g1CardTable.hpp" 33 #include "gc/g1/g1ThreadLocalData.hpp" 34 #include "gc/g1/heapRegion.hpp" 35 #include "interpreter/interp_masm.hpp" 36 #include "runtime/sharedRuntime.hpp" 37 38 #define __ masm-> 39 40 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 41 Register addr, Register count, RegSet saved_regs) { 42 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0; 43 if (!dest_uninitialized) { 44 __ push(saved_regs, sp); 45 if (count == c_rarg0) { 46 if (addr == c_rarg1) { 47 // exactly backwards!! 48 __ mov(rscratch1, c_rarg0); 49 __ mov(c_rarg0, c_rarg1); 50 __ mov(c_rarg1, rscratch1); 51 } else { 52 __ mov(c_rarg1, count); 53 __ mov(c_rarg0, addr); 54 } 55 } else { 56 __ mov(c_rarg0, addr); 57 __ mov(c_rarg1, count); 58 } 59 if (UseCompressedOops) { 60 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), 2); 61 } else { 62 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), 2); 63 } 64 __ pop(saved_regs, sp); 65 } 66 } 67 68 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 69 Register start, Register end, Register scratch, RegSet saved_regs) { 70 __ push(saved_regs, sp); 71 // must compute element count unless barrier set interface is changed (other platforms supply count) 72 assert_different_registers(start, end, scratch); 73 __ lea(scratch, Address(end, BytesPerHeapOop)); 74 __ sub(scratch, scratch, start); // subtract start to get #bytes 75 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count 76 __ mov(c_rarg0, start); 77 __ mov(c_rarg1, scratch); 78 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry), 2); 79 __ pop(saved_regs, sp); 80 } 81 82 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, 83 Register obj, 84 Register pre_val, 85 Register thread, 86 Register tmp, 87 bool tosca_live, 88 bool expand_call) { 89 // If expand_call is true then we expand the call_VM_leaf macro 90 // directly to skip generating the check by 91 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp. 92 93 assert(thread == rthread, "must be"); 94 95 Label done; 96 Label runtime; 97 98 assert_different_registers(obj, pre_val, tmp, rscratch1); 99 assert(pre_val != noreg && tmp != noreg, "expecting a register"); 100 101 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 102 Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 103 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 104 105 // Is marking active? 106 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 107 __ ldrw(tmp, in_progress); 108 } else { 109 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 110 __ ldrb(tmp, in_progress); 111 } 112 __ cbzw(tmp, done); 113 114 // Do we need to load the previous value? 115 if (obj != noreg) { 116 __ load_heap_oop(pre_val, Address(obj, 0)); 117 } 118 119 // Is the previous value null? 120 __ cbz(pre_val, done); 121 122 // Can we store original value in the thread's buffer? 123 // Is index == 0? 124 // (The index field is typed as size_t.) 125 126 __ ldr(tmp, index); // tmp := *index_adr 127 __ cbz(tmp, runtime); // tmp == 0? 128 // If yes, goto runtime 129 130 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize 131 __ str(tmp, index); // *index_adr := tmp 132 __ ldr(rscratch1, buffer); 133 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr 134 135 // Record the previous value 136 __ str(pre_val, Address(tmp, 0)); 137 __ b(done); 138 139 __ bind(runtime); 140 // save the live input values 141 RegSet saved = RegSet::of(pre_val); 142 if (tosca_live) saved += RegSet::of(r0); 143 if (obj != noreg) saved += RegSet::of(obj); 144 145 __ push(saved, sp); 146 147 // Calling the runtime using the regular call_VM_leaf mechanism generates 148 // code (generated by InterpreterMacroAssember::call_VM_leaf_base) 149 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL. 150 // 151 // If we care generating the pre-barrier without a frame (e.g. in the 152 // intrinsified Reference.get() routine) then ebp might be pointing to 153 // the caller frame and so this check will most likely fail at runtime. 154 // 155 // Expanding the call directly bypasses the generation of the check. 156 // So when we do not have have a full interpreter frame on the stack 157 // expand_call should be passed true. 158 159 if (expand_call) { 160 assert(pre_val != c_rarg1, "smashed arg"); 161 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); 162 } else { 163 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); 164 } 165 166 __ pop(saved, sp); 167 168 __ bind(done); 169 170 } 171 172 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, 173 Register store_addr, 174 Register new_val, 175 Register thread, 176 Register tmp, 177 Register tmp2) { 178 assert(thread == rthread, "must be"); 179 assert_different_registers(store_addr, new_val, thread, tmp, tmp2, 180 rscratch1); 181 assert(store_addr != noreg && new_val != noreg && tmp != noreg 182 && tmp2 != noreg, "expecting a register"); 183 184 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 185 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 186 187 BarrierSet* bs = BarrierSet::barrier_set(); 188 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 189 CardTable* ct = ctbs->card_table(); 190 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 191 192 Label done; 193 Label runtime; 194 195 // Does store cross heap regions? 196 197 __ eor(tmp, store_addr, new_val); 198 __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes); 199 __ cbz(tmp, done); 200 201 // crosses regions, storing NULL? 202 203 __ cbz(new_val, done); 204 205 // storing region crossing non-NULL, is card already dirty? 206 207 ExternalAddress cardtable((address) ct->byte_map_base()); 208 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 209 const Register card_addr = tmp; 210 211 __ lsr(card_addr, store_addr, CardTable::card_shift); 212 213 // get the address of the card 214 __ load_byte_map_base(tmp2); 215 __ add(card_addr, card_addr, tmp2); 216 __ ldrb(tmp2, Address(card_addr)); 217 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val()); 218 __ br(Assembler::EQ, done); 219 220 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 221 222 __ membar(Assembler::StoreLoad); 223 224 __ ldrb(tmp2, Address(card_addr)); 225 __ cbzw(tmp2, done); 226 227 // storing a region crossing, non-NULL oop, card is clean. 228 // dirty card and log. 229 230 __ strb(zr, Address(card_addr)); 231 232 __ ldr(rscratch1, queue_index); 233 __ cbz(rscratch1, runtime); 234 __ sub(rscratch1, rscratch1, wordSize); 235 __ str(rscratch1, queue_index); 236 237 __ ldr(tmp2, buffer); 238 __ str(card_addr, Address(tmp2, rscratch1)); 239 __ b(done); 240 241 __ bind(runtime); 242 // save the live input values 243 RegSet saved = RegSet::of(store_addr, new_val); 244 __ push(saved, sp); 245 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); 246 __ pop(saved, sp); 247 248 __ bind(done); 249 } 250 251 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 252 Register dst, Address src, Register tmp1, Register tmp_thread) { 253 bool on_oop = type == T_OBJECT || type == T_ARRAY; 254 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; 255 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; 256 bool on_reference = on_weak || on_phantom; 257 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread); 258 if (on_oop && on_reference) { 259 // Generate the G1 pre-barrier code to log the value of 260 // the referent field in an SATB buffer. 261 g1_write_barrier_pre(masm /* masm */, 262 noreg /* obj */, 263 dst /* pre_val */, 264 rthread /* thread */, 265 tmp1 /* tmp */, 266 true /* tosca_live */, 267 true /* expand_call */); 268 } 269 } 270 271 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 272 Address dst, Register val, Register tmp1, Register tmp2) { 273 // flatten object address if needed 274 if (dst.index() == noreg && dst.offset() == 0) { 275 if (dst.base() != r3) { 276 __ mov(r3, dst.base()); 277 } 278 } else { 279 __ lea(r3, dst); 280 } 281 282 g1_write_barrier_pre(masm, 283 r3 /* obj */, 284 tmp2 /* pre_val */, 285 rthread /* thread */, 286 tmp1 /* tmp */, 287 val != noreg /* tosca_live */, 288 false /* expand_call */); 289 290 if (val == noreg) { 291 __ store_heap_oop_null(Address(r3, 0)); 292 } else { 293 // G1 barrier needs uncompressed oop for region cross check. 294 Register new_val = val; 295 if (UseCompressedOops) { 296 new_val = rscratch2; 297 __ mov(new_val, val); 298 } 299 __ store_heap_oop(Address(r3, 0), val); 300 g1_write_barrier_post(masm, 301 r3 /* store_adr */, 302 new_val /* new_val */, 303 rthread /* thread */, 304 tmp1 /* tmp */, 305 tmp2 /* tmp2 */); 306 } 307 308 } 309 310 #ifdef COMPILER1 311 312 #undef __ 313 #define __ ce->masm()-> 314 315 void G1BarrierSetAssembler::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 316 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 317 // At this point we know that marking is in progress. 318 // If do_load() is true then we have to emit the 319 // load of the previous value; otherwise it has already 320 // been loaded into _pre_val. 321 322 __ bind(*stub->entry()); 323 324 assert(stub->pre_val()->is_register(), "Precondition."); 325 326 Register pre_val_reg = stub->pre_val()->as_register(); 327 328 if (stub->do_load()) { 329 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); 330 } 331 __ cbz(pre_val_reg, *stub->continuation()); 332 ce->store_parameter(stub->pre_val()->as_register(), 0); 333 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin())); 334 __ b(*stub->continuation()); 335 } 336 337 void G1BarrierSetAssembler::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 338 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 339 __ bind(*stub->entry()); 340 assert(stub->addr()->is_register(), "Precondition."); 341 assert(stub->new_val()->is_register(), "Precondition."); 342 Register new_val_reg = stub->new_val()->as_register(); 343 __ cbz(new_val_reg, *stub->continuation()); 344 ce->store_parameter(stub->addr()->as_pointer_register(), 0); 345 __ far_call(RuntimeAddress(bs->post_barrier_c1_runtime_code_blob()->code_begin())); 346 __ b(*stub->continuation()); 347 } 348 349 #undef __ 350 351 #define __ sasm-> 352 353 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 354 __ prologue("g1_pre_barrier", false); 355 356 // arg0 : previous value of memory 357 358 BarrierSet* bs = BarrierSet::barrier_set(); 359 360 const Register pre_val = r0; 361 const Register thread = rthread; 362 const Register tmp = rscratch1; 363 364 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 365 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 366 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 367 368 Label done; 369 Label runtime; 370 371 // Is marking still active? 372 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 373 __ ldrw(tmp, in_progress); 374 } else { 375 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 376 __ ldrb(tmp, in_progress); 377 } 378 __ cbzw(tmp, done); 379 380 // Can we store original value in the thread's buffer? 381 __ ldr(tmp, queue_index); 382 __ cbz(tmp, runtime); 383 384 __ sub(tmp, tmp, wordSize); 385 __ str(tmp, queue_index); 386 __ ldr(rscratch2, buffer); 387 __ add(tmp, tmp, rscratch2); 388 __ load_parameter(0, rscratch2); 389 __ str(rscratch2, Address(tmp, 0)); 390 __ b(done); 391 392 __ bind(runtime); 393 __ push_call_clobbered_registers(); 394 __ load_parameter(0, pre_val); 395 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); 396 __ pop_call_clobbered_registers(); 397 __ bind(done); 398 399 __ epilogue(); 400 } 401 402 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 403 __ prologue("g1_post_barrier", false); 404 405 // arg0: store_address 406 Address store_addr(rfp, 2*BytesPerWord); 407 408 BarrierSet* bs = BarrierSet::barrier_set(); 409 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 410 CardTable* ct = ctbs->card_table(); 411 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code"); 412 413 Label done; 414 Label runtime; 415 416 // At this point we know new_value is non-NULL and the new_value crosses regions. 417 // Must check to see if card is already dirty 418 419 const Register thread = rthread; 420 421 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 422 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 423 424 const Register card_offset = rscratch2; 425 // LR is free here, so we can use it to hold the byte_map_base. 426 const Register byte_map_base = lr; 427 428 assert_different_registers(card_offset, byte_map_base, rscratch1); 429 430 __ load_parameter(0, card_offset); 431 __ lsr(card_offset, card_offset, CardTable::card_shift); 432 __ load_byte_map_base(byte_map_base); 433 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 434 __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val()); 435 __ br(Assembler::EQ, done); 436 437 assert((int)CardTable::dirty_card_val() == 0, "must be 0"); 438 439 __ membar(Assembler::StoreLoad); 440 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 441 __ cbzw(rscratch1, done); 442 443 // storing region crossing non-NULL, card is clean. 444 // dirty card and log. 445 __ strb(zr, Address(byte_map_base, card_offset)); 446 447 // Convert card offset into an address in card_addr 448 Register card_addr = card_offset; 449 __ add(card_addr, byte_map_base, card_addr); 450 451 __ ldr(rscratch1, queue_index); 452 __ cbz(rscratch1, runtime); 453 __ sub(rscratch1, rscratch1, wordSize); 454 __ str(rscratch1, queue_index); 455 456 // Reuse LR to hold buffer_addr 457 const Register buffer_addr = lr; 458 459 __ ldr(buffer_addr, buffer); 460 __ str(card_addr, Address(buffer_addr, rscratch1)); 461 __ b(done); 462 463 __ bind(runtime); 464 __ push_call_clobbered_registers(); 465 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); 466 __ pop_call_clobbered_registers(); 467 __ bind(done); 468 __ epilogue(); 469 } 470 471 #undef __ 472 473 #endif // COMPILER1