1 /* 2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2018, SAP SE. All rights reserved. 4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 5 * 6 * This code is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 only, as 8 * published by the Free Software Foundation. 9 * 10 * This code is distributed in the hope that it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * version 2 for more details (a copy is included in the LICENSE file that 14 * accompanied this code). 15 * 16 * You should have received a copy of the GNU General Public License version 17 * 2 along with this work; if not, write to the Free Software Foundation, 18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 19 * 20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 21 * or visit www.oracle.com if you need additional information or have any 22 * questions. 23 * 24 */ 25 26 #include "precompiled.hpp" 27 #include "asm/macroAssembler.inline.hpp" 28 #include "registerSaver_s390.hpp" 29 #include "c1/c1_LIRAssembler.hpp" 30 #include "c1/c1_MacroAssembler.hpp" 31 #include "gc/g1/c1/g1BarrierSetC1.hpp" 32 #include "gc/g1/g1CardTable.hpp" 33 #include "gc/g1/g1BarrierSet.hpp" 34 #include "gc/g1/g1BarrierSetAssembler.hpp" 35 #include "gc/g1/g1ThreadLocalData.hpp" 36 #include "gc/g1/heapRegion.hpp" 37 #include "interpreter/interp_masm.hpp" 38 #include "runtime/sharedRuntime.hpp" 39 40 #define __ masm-> 41 42 #define BLOCK_COMMENT(str) if (PrintAssembly) __ block_comment(str) 43 44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators, 45 Register addr, Register count) { 46 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0; 47 48 // With G1, don't generate the call if we statically know that the target is uninitialized. 49 if (!dest_uninitialized) { 50 // Is marking active? 51 Label filtered; 52 assert_different_registers(addr, Z_R0_scratch); // would be destroyed by push_frame() 53 assert_different_registers(count, Z_R0_scratch); // would be destroyed by push_frame() 54 Register Rtmp1 = Z_R0_scratch; 55 const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); 56 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 57 __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset)); 58 } else { 59 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 60 __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset)); 61 } 62 __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently. 63 64 RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame. 65 66 if (UseCompressedOops) { 67 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), addr, count); 68 } else { 69 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), addr, count); 70 } 71 72 RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers); 73 74 __ bind(filtered); 75 } 76 } 77 78 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, 79 Register addr, Register count, bool do_return) { 80 address entry_point = CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry); 81 if (!do_return) { 82 assert_different_registers(addr, Z_R0_scratch); // would be destroyed by push_frame() 83 assert_different_registers(count, Z_R0_scratch); // would be destroyed by push_frame() 84 RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame. 85 __ call_VM_leaf(entry_point, addr, count); 86 RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers); 87 } else { 88 // Tail call: call c and return to stub caller. 89 __ lgr_if_needed(Z_ARG1, addr); 90 __ lgr_if_needed(Z_ARG2, count); 91 __ load_const(Z_R1, entry_point); 92 __ z_br(Z_R1); // Branch without linking, callee will return to stub caller. 93 } 94 } 95 96 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 97 const Address& src, Register dst, Register tmp1, Register tmp2, Label *is_null) { 98 bool on_oop = type == T_OBJECT || type == T_ARRAY; 99 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0; 100 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0; 101 bool on_reference = on_weak || on_phantom; 102 Label done; 103 if (on_oop && on_reference && is_null == NULL) { is_null = &done; } 104 ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, is_null); 105 if (on_oop && on_reference) { 106 // Generate the G1 pre-barrier code to log the value of 107 // the referent field in an SATB buffer. 108 g1_write_barrier_pre(masm, decorators | OOP_NOT_NULL, 109 NULL /* obj */, 110 dst /* pre_val */, 111 noreg/* preserve */ , 112 tmp1, tmp2 /* tmp */, 113 true /* pre_val_needed */); 114 } 115 __ bind(done); 116 } 117 118 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators, 119 const Address* obj, 120 Register Rpre_val, // Ideally, this is a non-volatile register. 121 Register Rval, // Will be preserved. 122 Register Rtmp1, // If Rpre_val is volatile, either Rtmp1 123 Register Rtmp2, // or Rtmp2 has to be non-volatile. 124 bool pre_val_needed // Save Rpre_val across runtime call, caller uses it. 125 ) { 126 127 bool not_null = (decorators & OOP_NOT_NULL) != 0, 128 preloaded = obj == NULL; 129 130 const Register Robj = obj ? obj->base() : noreg, 131 Roff = obj ? obj->index() : noreg; 132 const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); 133 const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()); 134 const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()); 135 assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp<i> must be Z_R0!! 136 assert_different_registers(Robj, Z_R0_scratch); // Used for addressing. Furthermore, push_frame destroys Z_R0!! 137 assert_different_registers(Rval, Z_R0_scratch); // push_frame destroys Z_R0!! 138 139 Label callRuntime, filtered; 140 141 BLOCK_COMMENT("g1_write_barrier_pre {"); 142 143 // Is marking active? 144 // Note: value is loaded for test purposes only. No further use here. 145 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 146 __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset)); 147 } else { 148 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 149 __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset)); 150 } 151 __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently. 152 153 assert(Rpre_val != noreg, "must have a real register"); 154 155 156 // If an object is given, we need to load the previous value into Rpre_val. 157 if (obj) { 158 // Load the previous value... 159 if (UseCompressedOops) { 160 __ z_llgf(Rpre_val, *obj); 161 } else { 162 __ z_lg(Rpre_val, *obj); 163 } 164 } 165 166 // Is the previous value NULL? 167 // If so, we don't need to record it and we're done. 168 // Note: pre_val is loaded, decompressed and stored (directly or via runtime call). 169 // Register contents is preserved across runtime call if caller requests to do so. 170 if (preloaded && not_null) { 171 #ifdef ASSERT 172 __ z_ltgr(Rpre_val, Rpre_val); 173 __ asm_assert_ne("null oop not allowed (G1 pre)", 0x321); // Checked by caller. 174 #endif 175 } else { 176 __ z_ltgr(Rpre_val, Rpre_val); 177 __ z_bre(filtered); // previous value is NULL, so we don't need to record it. 178 } 179 180 // Decode the oop now. We know it's not NULL. 181 if (Robj != noreg && UseCompressedOops) { 182 __ oop_decoder(Rpre_val, Rpre_val, /*maybeNULL=*/false); 183 } 184 185 // OK, it's not filtered, so we'll need to call enqueue. 186 187 // We can store the original value in the thread's buffer 188 // only if index > 0. Otherwise, we need runtime to handle. 189 // (The index field is typed as size_t.) 190 Register Rbuffer = Rtmp1, Rindex = Rtmp2; 191 assert_different_registers(Rbuffer, Rindex, Rpre_val); 192 193 __ z_lg(Rbuffer, buffer_offset, Z_thread); 194 195 __ load_and_test_long(Rindex, Address(Z_thread, index_offset)); 196 __ z_bre(callRuntime); // If index == 0, goto runtime. 197 198 __ add2reg(Rindex, -wordSize); // Decrement index. 199 __ z_stg(Rindex, index_offset, Z_thread); 200 201 // Record the previous value. 202 __ z_stg(Rpre_val, 0, Rbuffer, Rindex); 203 __ z_bru(filtered); // We are done. 204 205 Rbuffer = noreg; // end of life 206 Rindex = noreg; // end of life 207 208 __ bind(callRuntime); 209 210 // Save some registers (inputs and result) over runtime call 211 // by spilling them into the top frame. 212 if (Robj != noreg && Robj->is_volatile()) { 213 __ z_stg(Robj, Robj->encoding()*BytesPerWord, Z_SP); 214 } 215 if (Roff != noreg && Roff->is_volatile()) { 216 __ z_stg(Roff, Roff->encoding()*BytesPerWord, Z_SP); 217 } 218 if (Rval != noreg && Rval->is_volatile()) { 219 __ z_stg(Rval, Rval->encoding()*BytesPerWord, Z_SP); 220 } 221 222 // Save Rpre_val (result) over runtime call. 223 Register Rpre_save = Rpre_val; 224 if ((Rpre_val == Z_R0_scratch) || (pre_val_needed && Rpre_val->is_volatile())) { 225 guarantee(!Rtmp1->is_volatile() || !Rtmp2->is_volatile(), "oops!"); 226 Rpre_save = !Rtmp1->is_volatile() ? Rtmp1 : Rtmp2; 227 } 228 __ lgr_if_needed(Rpre_save, Rpre_val); 229 230 // Push frame to protect top frame with return pc and spilled register values. 231 __ save_return_pc(); 232 __ push_frame_abi160(0); // Will use Z_R0 as tmp. 233 234 // Rpre_val may be destroyed by push_frame(). 235 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), Rpre_save, Z_thread); 236 237 __ pop_frame(); 238 __ restore_return_pc(); 239 240 // Restore spilled values. 241 if (Robj != noreg && Robj->is_volatile()) { 242 __ z_lg(Robj, Robj->encoding()*BytesPerWord, Z_SP); 243 } 244 if (Roff != noreg && Roff->is_volatile()) { 245 __ z_lg(Roff, Roff->encoding()*BytesPerWord, Z_SP); 246 } 247 if (Rval != noreg && Rval->is_volatile()) { 248 __ z_lg(Rval, Rval->encoding()*BytesPerWord, Z_SP); 249 } 250 if (pre_val_needed && Rpre_val->is_volatile()) { 251 __ lgr_if_needed(Rpre_val, Rpre_save); 252 } 253 254 __ bind(filtered); 255 BLOCK_COMMENT("} g1_write_barrier_pre"); 256 } 257 258 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, Register Rstore_addr, Register Rnew_val, 259 Register Rtmp1, Register Rtmp2, Register Rtmp3) { 260 bool not_null = (decorators & OOP_NOT_NULL) != 0; 261 262 assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); // Most probably, Rnew_val == Rtmp3. 263 264 Label callRuntime, filtered; 265 266 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set()); 267 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code"); 268 269 BLOCK_COMMENT("g1_write_barrier_post {"); 270 271 // Does store cross heap regions? 272 // It does if the two addresses specify different grain addresses. 273 if (G1RSBarrierRegionFilter) { 274 if (VM_Version::has_DistinctOpnds()) { 275 __ z_xgrk(Rtmp1, Rstore_addr, Rnew_val); 276 } else { 277 __ z_lgr(Rtmp1, Rstore_addr); 278 __ z_xgr(Rtmp1, Rnew_val); 279 } 280 __ z_srag(Rtmp1, Rtmp1, HeapRegion::LogOfHRGrainBytes); 281 __ z_bre(filtered); 282 } 283 284 // Crosses regions, storing NULL? 285 if (not_null) { 286 #ifdef ASSERT 287 __ z_ltgr(Rnew_val, Rnew_val); 288 __ asm_assert_ne("null oop not allowed (G1 post)", 0x322); // Checked by caller. 289 #endif 290 } else { 291 __ z_ltgr(Rnew_val, Rnew_val); 292 __ z_bre(filtered); 293 } 294 295 Rnew_val = noreg; // end of lifetime 296 297 // Storing region crossing non-NULL, is card already dirty? 298 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code"); 299 assert_different_registers(Rtmp1, Rtmp2, Rtmp3); 300 // Make sure not to use Z_R0 for any of these registers. 301 Register Rcard_addr = (Rtmp1 != Z_R0_scratch) ? Rtmp1 : Rtmp3; 302 Register Rbase = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp3; 303 304 // calculate address of card 305 __ load_const_optimized(Rbase, (address)ct->card_table()->byte_map_base()); // Card table base. 306 __ z_srlg(Rcard_addr, Rstore_addr, CardTable::card_shift); // Index into card table. 307 __ z_algr(Rcard_addr, Rbase); // Explicit calculation needed for cli. 308 Rbase = noreg; // end of lifetime 309 310 // Filter young. 311 assert((unsigned int)G1CardTable::g1_young_card_val() <= 255, "otherwise check this code"); 312 __ z_cli(0, Rcard_addr, G1CardTable::g1_young_card_val()); 313 __ z_bre(filtered); 314 315 // Check the card value. If dirty, we're done. 316 // This also avoids false sharing of the (already dirty) card. 317 __ z_sync(); // Required to support concurrent cleaning. 318 assert((unsigned int)G1CardTable::dirty_card_val() <= 255, "otherwise check this code"); 319 __ z_cli(0, Rcard_addr, G1CardTable::dirty_card_val()); // Reload after membar. 320 __ z_bre(filtered); 321 322 // Storing a region crossing, non-NULL oop, card is clean. 323 // Dirty card and log. 324 __ z_mvi(0, Rcard_addr, G1CardTable::dirty_card_val()); 325 326 Register Rcard_addr_x = Rcard_addr; 327 Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1; 328 Register Rqueue_buf = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1; 329 const int qidx_off = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()); 330 const int qbuf_off = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()); 331 if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) { 332 Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0. 333 } 334 __ lgr_if_needed(Rcard_addr_x, Rcard_addr); 335 336 __ load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off)); 337 __ z_bre(callRuntime); // Index == 0 then jump to runtime. 338 339 __ z_lg(Rqueue_buf, qbuf_off, Z_thread); 340 341 __ add2reg(Rqueue_index, -wordSize); // Decrement index. 342 __ z_stg(Rqueue_index, qidx_off, Z_thread); 343 344 __ z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card. 345 __ z_bru(filtered); 346 347 __ bind(callRuntime); 348 349 // TODO: do we need a frame? Introduced to be on the safe side. 350 bool needs_frame = true; 351 __ lgr_if_needed(Rcard_addr, Rcard_addr_x); // copy back asap. push_frame will destroy Z_R0_scratch! 352 353 // VM call need frame to access(write) O register. 354 if (needs_frame) { 355 __ save_return_pc(); 356 __ push_frame_abi160(0); // Will use Z_R0 as tmp on old CPUs. 357 } 358 359 // Save the live input values. 360 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, Z_thread); 361 362 if (needs_frame) { 363 __ pop_frame(); 364 __ restore_return_pc(); 365 } 366 367 __ bind(filtered); 368 369 BLOCK_COMMENT("} g1_write_barrier_post"); 370 } 371 372 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, 373 const Address& dst, Register val, Register tmp1, Register tmp2, Register tmp3) { 374 bool on_array = (decorators & IN_HEAP_ARRAY) != 0; 375 bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; 376 bool precise = on_array || on_anonymous; 377 // Load and record the previous value. 378 g1_write_barrier_pre(masm, decorators, &dst, tmp3, val, tmp1, tmp2, false); 379 380 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3); 381 382 // No need for post barrier if storing NULL 383 if (val != noreg) { 384 const Register base = dst.base(), 385 idx = dst.index(); 386 const intptr_t disp = dst.disp(); 387 if (precise && (disp != 0 || idx != noreg)) { 388 __ add2reg_with_index(base, disp, idx, base); 389 } 390 g1_write_barrier_post(masm, decorators, base, val, tmp1, tmp2, tmp3); 391 } 392 } 393 394 void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2) { 395 NearLabel Ldone, Lnot_weak; 396 __ z_ltgr(tmp1, value); 397 __ z_bre(Ldone); // Use NULL result as-is. 398 399 __ z_nill(value, ~JNIHandles::weak_tag_mask); 400 __ z_lg(value, 0, value); // Resolve (untagged) jobject. 401 402 __ z_tmll(tmp1, JNIHandles::weak_tag_mask); // Test for jweak tag. 403 __ z_braz(Lnot_weak); 404 __ verify_oop(value); 405 DecoratorSet decorators = IN_ROOT | ON_PHANTOM_OOP_REF; 406 g1_write_barrier_pre(masm, decorators, (const Address*)NULL, value, noreg, tmp1, tmp2, true); 407 __ bind(Lnot_weak); 408 __ verify_oop(value); 409 __ bind(Ldone); 410 } 411 412 #ifdef COMPILER1 413 414 #undef __ 415 #define __ ce->masm()-> 416 417 void G1BarrierSetAssembler::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) { 418 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 419 // At this point we know that marking is in progress. 420 // If do_load() is true then we have to emit the 421 // load of the previous value; otherwise it has already 422 // been loaded into _pre_val. 423 __ bind(*stub->entry()); 424 ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots. 425 assert(stub->pre_val()->is_register(), "Precondition."); 426 427 Register pre_val_reg = stub->pre_val()->as_register(); 428 429 if (stub->do_load()) { 430 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/); 431 } 432 433 __ z_ltgr(Z_R1_scratch, pre_val_reg); // Pass oop in Z_R1_scratch to Runtime1::g1_pre_barrier_slow_id. 434 __ branch_optimized(Assembler::bcondZero, *stub->continuation()); 435 ce->emit_call_c(bs->pre_barrier_c1_runtime_code_blob()->code_begin()); 436 __ branch_optimized(Assembler::bcondAlways, *stub->continuation()); 437 } 438 439 void G1BarrierSetAssembler::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) { 440 G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1(); 441 __ bind(*stub->entry()); 442 ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots. 443 assert(stub->addr()->is_register(), "Precondition."); 444 assert(stub->new_val()->is_register(), "Precondition."); 445 Register new_val_reg = stub->new_val()->as_register(); 446 __ z_ltgr(new_val_reg, new_val_reg); 447 __ branch_optimized(Assembler::bcondZero, *stub->continuation()); 448 __ z_lgr(Z_R1_scratch, stub->addr()->as_pointer_register()); 449 ce->emit_call_c(bs->post_barrier_c1_runtime_code_blob()->code_begin()); 450 __ branch_optimized(Assembler::bcondAlways, *stub->continuation()); 451 } 452 453 #undef __ 454 455 #define __ sasm-> 456 457 static OopMap* save_volatile_registers(StubAssembler* sasm, Register return_pc = Z_R14) { 458 __ block_comment("save_volatile_registers"); 459 RegisterSaver::RegisterSet reg_set = RegisterSaver::all_volatile_registers; 460 int frame_size_in_slots = RegisterSaver::live_reg_frame_size(reg_set) / VMRegImpl::stack_slot_size; 461 sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word); 462 return RegisterSaver::save_live_registers(sasm, reg_set, return_pc); 463 } 464 465 static void restore_volatile_registers(StubAssembler* sasm) { 466 __ block_comment("restore_volatile_registers"); 467 RegisterSaver::RegisterSet reg_set = RegisterSaver::all_volatile_registers; 468 RegisterSaver::restore_live_registers(sasm, reg_set); 469 } 470 471 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) { 472 // Z_R1_scratch: previous value of memory 473 474 BarrierSet* bs = BarrierSet::barrier_set(); 475 __ set_info("g1_pre_barrier_slow_id", false); 476 477 Register pre_val = Z_R1_scratch; 478 Register tmp = Z_R6; // Must be non-volatile because it is used to save pre_val. 479 Register tmp2 = Z_R7; 480 481 Label refill, restart, marking_not_active; 482 int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()); 483 int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()); 484 int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()); 485 486 // Save tmp registers (see assertion in G1PreBarrierStub::emit_code()). 487 __ z_stg(tmp, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 488 __ z_stg(tmp2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 489 490 // Is marking still active? 491 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) { 492 __ load_and_test_int(tmp, Address(Z_thread, satb_q_active_byte_offset)); 493 } else { 494 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption"); 495 __ load_and_test_byte(tmp, Address(Z_thread, satb_q_active_byte_offset)); 496 } 497 __ z_bre(marking_not_active); // Activity indicator is zero, so there is no marking going on currently. 498 499 __ bind(restart); 500 // Load the index into the SATB buffer. SATBMarkQueue::_index is a 501 // size_t so ld_ptr is appropriate. 502 __ z_ltg(tmp, satb_q_index_byte_offset, Z_R0, Z_thread); 503 504 // index == 0? 505 __ z_brz(refill); 506 507 __ z_lg(tmp2, satb_q_buf_byte_offset, Z_thread); 508 __ add2reg(tmp, -oopSize); 509 510 __ z_stg(pre_val, 0, tmp, tmp2); // [_buf + index] := <address_of_card> 511 __ z_stg(tmp, satb_q_index_byte_offset, Z_thread); 512 513 __ bind(marking_not_active); 514 // Restore tmp registers (see assertion in G1PreBarrierStub::emit_code()). 515 __ z_lg(tmp, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 516 __ z_lg(tmp2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 517 __ z_br(Z_R14); 518 519 __ bind(refill); 520 save_volatile_registers(sasm); 521 __ z_lgr(tmp, pre_val); // save pre_val 522 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SATBMarkQueueSet::handle_zero_index_for_thread), 523 Z_thread); 524 __ z_lgr(pre_val, tmp); // restore pre_val 525 restore_volatile_registers(sasm); 526 __ z_bru(restart); 527 } 528 529 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) { 530 // Z_R1_scratch: oop address, address of updated memory slot 531 532 BarrierSet* bs = BarrierSet::barrier_set(); 533 __ set_info("g1_post_barrier_slow_id", false); 534 535 Register addr_oop = Z_R1_scratch; 536 Register addr_card = Z_R1_scratch; 537 Register r1 = Z_R6; // Must be saved/restored. 538 Register r2 = Z_R7; // Must be saved/restored. 539 Register cardtable = r1; // Must be non-volatile, because it is used to save addr_card. 540 CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs); 541 CardTable* ct = ctbs->card_table(); 542 jbyte* byte_map_base = ct->byte_map_base(); 543 544 // Save registers used below (see assertion in G1PreBarrierStub::emit_code()). 545 __ z_stg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 546 547 Label not_already_dirty, restart, refill, young_card; 548 549 // Calculate address of card corresponding to the updated oop slot. 550 AddressLiteral rs(byte_map_base); 551 __ z_srlg(addr_card, addr_oop, CardTable::card_shift); 552 addr_oop = noreg; // dead now 553 __ load_const_optimized(cardtable, rs); // cardtable := <card table base> 554 __ z_agr(addr_card, cardtable); // addr_card := addr_oop>>card_shift + cardtable 555 556 __ z_cli(0, addr_card, (int)G1CardTable::g1_young_card_val()); 557 __ z_bre(young_card); 558 559 __ z_sync(); // Required to support concurrent cleaning. 560 561 __ z_cli(0, addr_card, (int)CardTable::dirty_card_val()); 562 __ z_brne(not_already_dirty); 563 564 __ bind(young_card); 565 // We didn't take the branch, so we're already dirty: restore 566 // used registers and return. 567 __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 568 __ z_br(Z_R14); 569 570 // Not dirty. 571 __ bind(not_already_dirty); 572 573 // First, dirty it: [addr_card] := 0 574 __ z_mvi(0, addr_card, CardTable::dirty_card_val()); 575 576 Register idx = cardtable; // Must be non-volatile, because it is used to save addr_card. 577 Register buf = r2; 578 cardtable = noreg; // now dead 579 580 // Save registers used below (see assertion in G1PreBarrierStub::emit_code()). 581 __ z_stg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 582 583 ByteSize dirty_card_q_index_byte_offset = G1ThreadLocalData::dirty_card_queue_index_offset(); 584 ByteSize dirty_card_q_buf_byte_offset = G1ThreadLocalData::dirty_card_queue_buffer_offset(); 585 586 __ bind(restart); 587 588 // Get the index into the update buffer. DirtyCardQueue::_index is 589 // a size_t so z_ltg is appropriate here. 590 __ z_ltg(idx, Address(Z_thread, dirty_card_q_index_byte_offset)); 591 592 // index == 0? 593 __ z_brz(refill); 594 595 __ z_lg(buf, Address(Z_thread, dirty_card_q_buf_byte_offset)); 596 __ add2reg(idx, -oopSize); 597 598 __ z_stg(addr_card, 0, idx, buf); // [_buf + index] := <address_of_card> 599 __ z_stg(idx, Address(Z_thread, dirty_card_q_index_byte_offset)); 600 // Restore killed registers and return. 601 __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 602 __ z_lg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP); 603 __ z_br(Z_R14); 604 605 __ bind(refill); 606 save_volatile_registers(sasm); 607 __ z_lgr(idx, addr_card); // Save addr_card, tmp3 must be non-volatile. 608 __ call_VM_leaf(CAST_FROM_FN_PTR(address, DirtyCardQueueSet::handle_zero_index_for_thread), 609 Z_thread); 610 __ z_lgr(addr_card, idx); 611 restore_volatile_registers(sasm); // Restore addr_card. 612 __ z_bru(restart); 613 } 614 615 #undef __ 616 617 #endif // COMPILER1