1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.inline.hpp" 27 #include "gc/g1/g1BarrierSet.hpp" 28 #include "gc/g1/g1BarrierSetAssembler.hpp" 29 #include "gc/g1/g1CardTable.hpp" 30 #include "gc/g1/heapRegion.hpp" 31 #include "interpreter/interp_masm.hpp" 32 #include "runtime/sharedRuntime.hpp" 33 #include "runtime/thread.hpp" 34 #include "utilities/macros.hpp" 35 36 #define __ masm-> 37 38 #ifdef PRODUCT 39 #define BLOCK_COMMENT(str) /* nothing */ 40 #else 41 #define BLOCK_COMMENT(str) __ block_comment(str) 42 #endif 43 44 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") 45 46 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 47 Register addr, Register count, int callee_saved_regs) { 48 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0; 49 if (!dest_uninitialized) { 50 assert( addr->encoding() < callee_saved_regs, "addr must be saved"); 51 assert(count->encoding() < callee_saved_regs, "count must be saved"); 52 53 BLOCK_COMMENT("PreBarrier"); 54 55 #ifdef AARCH64 56 callee_saved_regs = align_up(callee_saved_regs, 2); 57 for (int i = 0; i < callee_saved_regs; i += 2) { 58 __ raw_push(as_Register(i), as_Register(i+1)); 59 } 60 #else 61 RegisterSet saved_regs = RegisterSet(R0, as_Register(callee_saved_regs-1)); 62 __ push(saved_regs | R9ifScratched); 63 #endif // AARCH64 64 65 if (addr != R0) { 66 assert_different_registers(count, R0); 67 __ mov(R0, addr); 68 } 69 #ifdef AARCH64 70 __ zero_extend(R1, count, 32); // G1BarrierSet::write_ref_array_pre_*_entry takes size_t 71 #else 72 if (count != R1) { 73 __ mov(R1, count); 74 } 75 #endif // AARCH64 76 77 if (UseCompressedOops) { 78 __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry)); 79 } else { 80 __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry)); 81 } 82 83 #ifdef AARCH64 84 for (int i = callee_saved_regs - 2; i >= 0; i -= 2) { 85 __ raw_pop(as_Register(i), as_Register(i+1)); 86 } 87 #else 88 __ pop(saved_regs | R9ifScratched); 89 #endif // AARCH64 90 } 91 } 92 93 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 94 Register addr, Register count, Register tmp) { 95 96 BLOCK_COMMENT("G1PostBarrier"); 97 if (addr != R0) { 98 assert_different_registers(count, R0); 99 __ mov(R0, addr); 100 } 101 #ifdef AARCH64 102 __ zero_extend(R1, count, 32); // G1BarrierSet::write_ref_array_post_entry takes size_t 103 #else 104 if (count != R1) { 105 __ mov(R1, count); 106 } 107 #if R9_IS_SCRATCHED 108 // Safer to save R9 here since callers may have been written 109 // assuming R9 survives. This is suboptimal but is not in 110 // general worth optimizing for the few platforms where R9 111 // is scratched. Note that the optimization might not be to 112 // difficult for this particular call site. 113 __ push(R9); 114 #endif // !R9_IS_SCRATCHED 115 #endif // !AARCH64 116 __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry)); 117 #ifndef AARCH64 118 #if R9_IS_SCRATCHED 119 __ pop(R9); 120 #endif // !R9_IS_SCRATCHED 121 #endif // !AARCH64 122 } 123 124 #ifdef COMPILER1 125 126 #undef __ 127 #define __ ce->masm()-> 128 129 void G1BarrierSetAssembler::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 130 // At this point we know that marking is in progress. 131 // If do_load() is true then we have to emit the 132 // load of the previous value; otherwise it has already 133 // been loaded into _pre_val. 134 135 __ bind(*stub->entry()); 136 assert(stub->pre_val()->is_register(), "Precondition."); 137 138 Register pre_val_reg = stub->pre_val()->as_register(); 139 140 if (stub->do_load()) { 141 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); 142 } 143 144 __ cbz(pre_val_reg, *stub->continuation()); 145 ce->verify_reserved_argument_area_size(1); 146 __ str(pre_val_reg, Address(SP)); 147 __ call(Runtime1::entry_for(Runtime1::g1_pre_barrier_slow_id), relocInfo::runtime_call_type); 148 149 __ b(*stub->continuation()); 150 } 151 152 void G1BarrierSetAssembler::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 153 __ bind(*stub->entry()); 154 assert(stub->addr()->is_register(), "Precondition."); 155 assert(stub->new_val()->is_register(), "Precondition."); 156 Register new_val_reg = stub->new_val()->as_register(); 157 __ cbz(new_val_reg, *stub->continuation()); 158 ce->verify_reserved_argument_area_size(1); 159 __ str(stub->addr()->as_pointer_register(), Address(SP)); 160 __ call(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id), relocInfo::runtime_call_type); 161 __ b(*stub->continuation()); 162 } 163 164 #undef __ 165 #define __ sasm-> 166 167 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 168 // Input: 169 // - pre_val pushed on the stack 170 171 __ set_info("g1_pre_barrier_slow_id", dont_gc_arguments); 172 173 BarrierSet* bs = BarrierSet::barrier_set(); 174 if (bs->kind() != BarrierSet::G1BarrierSet) { 175 __ mov(R0, (int)id); 176 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), R0); 177 __ should_not_reach_here(); 178 break; 179 } 180 181 // save at least the registers that need saving if the runtime is called 182 #ifdef AARCH64 183 __ raw_push(R0, R1); 184 __ raw_push(R2, R3); 185 const int nb_saved_regs = 4; 186 #else // AARCH64 187 const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR); 188 const int nb_saved_regs = 6; 189 assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs"); 190 __ push(saved_regs); 191 #endif // AARCH64 192 193 const Register r_pre_val_0 = R0; // must be R0, to be ready for the runtime call 194 const Register r_index_1 = R1; 195 const Register r_buffer_2 = R2; 196 197 Address queue_active(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset())); 198 Address queue_index(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset())); 199 Address buffer(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset())); 200 201 Label done; 202 Label runtime; 203 204 // Is marking still active? 205 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 206 __ ldrb(R1, queue_active); 207 __ cbz(R1, done); 208 209 __ ldr(r_index_1, queue_index); 210 __ ldr(r_pre_val_0, Address(SP, nb_saved_regs*wordSize)); 211 __ ldr(r_buffer_2, buffer); 212 213 __ subs(r_index_1, r_index_1, wordSize); 214 __ b(runtime, lt); 215 216 __ str(r_index_1, queue_index); 217 __ str(r_pre_val_0, Address(r_buffer_2, r_index_1)); 218 219 __ bind(done); 220 221 #ifdef AARCH64 222 __ raw_pop(R2, R3); 223 __ raw_pop(R0, R1); 224 #else // AARCH64 225 __ pop(saved_regs); 226 #endif // AARCH64 227 228 __ ret(); 229 230 __ bind(runtime); 231 232 __ save_live_registers(); 233 234 assert(r_pre_val_0 == c_rarg0, "pre_val should be in R0"); 235 __ mov(c_rarg1, Rthread); 236 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, c_rarg1); 237 238 __ restore_live_registers_without_return(); 239 240 __ b(done); 241 } 242 243 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 244 // Input: 245 // - store_addr, pushed on the stack 246 247 __ set_info("g1_post_barrier_slow_id", dont_gc_arguments); 248 249 BarrierSet* bs = BarrierSet::barrier_set(); 250 if (bs->kind() != BarrierSet::G1BarrierSet) { 251 __ mov(R0, (int)id); 252 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), R0); 253 __ should_not_reach_here(); 254 break; 255 } 256 257 Label done; 258 Label recheck; 259 Label runtime; 260 261 Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset())); 262 Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset())); 263 264 AddressLiteral cardtable(ci_card_table_address_as<address>(), relocInfo::none); 265 266 // save at least the registers that need saving if the runtime is called 267 #ifdef AARCH64 268 __ raw_push(R0, R1); 269 __ raw_push(R2, R3); 270 const int nb_saved_regs = 4; 271 #else // AARCH64 272 const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR); 273 const int nb_saved_regs = 6; 274 assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs"); 275 __ push(saved_regs); 276 #endif // AARCH64 277 278 const Register r_card_addr_0 = R0; // must be R0 for the slow case 279 const Register r_obj_0 = R0; 280 const Register r_card_base_1 = R1; 281 const Register r_tmp2 = R2; 282 const Register r_index_2 = R2; 283 const Register r_buffer_3 = R3; 284 const Register tmp1 = Rtemp; 285 286 __ ldr(r_obj_0, Address(SP, nb_saved_regs*wordSize)); 287 // Note: there is a comment in x86 code about not using 288 // ExternalAddress / lea, due to relocation not working 289 // properly for that address. Should be OK for arm, where we 290 // explicitly specify that 'cardtable' has a relocInfo::none 291 // type. 292 __ lea(r_card_base_1, cardtable); 293 __ add(r_card_addr_0, r_card_base_1, AsmOperand(r_obj_0, lsr, CardTable::card_shift)); 294 295 // first quick check without barrier 296 __ ldrb(r_tmp2, Address(r_card_addr_0)); 297 298 __ cmp(r_tmp2, (int)G1CardTable::g1_young_card_val()); 299 __ b(recheck, ne); 300 301 __ bind(done); 302 303 #ifdef AARCH64 304 __ raw_pop(R2, R3); 305 __ raw_pop(R0, R1); 306 #else // AARCH64 307 __ pop(saved_regs); 308 #endif // AARCH64 309 310 __ ret(); 311 312 __ bind(recheck); 313 314 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp1); 315 316 // reload card state after the barrier that ensures the stored oop was visible 317 __ ldrb(r_tmp2, Address(r_card_addr_0)); 318 319 assert(CardTable::dirty_card_val() == 0, "adjust this code"); 320 __ cbz(r_tmp2, done); 321 322 // storing region crossing non-NULL, card is clean. 323 // dirty card and log. 324 325 assert(0 == (int)CardTable::dirty_card_val(), "adjust this code"); 326 if ((ci_card_table_address_as<intptr_t>() & 0xff) == 0) { 327 // Card table is aligned so the lowest byte of the table address base is zero. 328 __ strb(r_card_base_1, Address(r_card_addr_0)); 329 } else { 330 __ strb(__ zero_register(r_tmp2), Address(r_card_addr_0)); 331 } 332 333 __ ldr(r_index_2, queue_index); 334 __ ldr(r_buffer_3, buffer); 335 336 __ subs(r_index_2, r_index_2, wordSize); 337 __ b(runtime, lt); // go to runtime if now negative 338 339 __ str(r_index_2, queue_index); 340 341 __ str(r_card_addr_0, Address(r_buffer_3, r_index_2)); 342 343 __ b(done); 344 345 __ bind(runtime); 346 347 __ save_live_registers(); 348 349 assert(r_card_addr_0 == c_rarg0, "card_addr should be in R0"); 350 __ mov(c_rarg1, Rthread); 351 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), c_rarg0, c_rarg1); 352 353 __ restore_live_registers_without_return(); 354 355 __ b(done); 356 } 357 358 #undef __ 359 360 #endif // COMPILER1