1 /*
  2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2018, SAP SE. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 #include "asm/macroAssembler.inline.hpp"
 28 #include "registerSaver_s390.hpp"
 29 #include "c1/c1_LIRAssembler.hpp"
 30 #include "c1/c1_MacroAssembler.hpp"
 31 #include "gc/g1/c1/g1BarrierSetC1.hpp"
 32 #include "gc/g1/g1CardTable.hpp"
 33 #include "gc/g1/g1BarrierSet.hpp"
 34 #include "gc/g1/g1BarrierSetAssembler.hpp"
 35 #include "gc/g1/g1ThreadLocalData.hpp"
 36 #include "gc/g1/heapRegion.hpp"
 37 #include "interpreter/interp_masm.hpp"
 38 #include "runtime/sharedRuntime.hpp"
 39 
 40 #define __ masm->
 41 
 42 #define BLOCK_COMMENT(str) if (PrintAssembly) __ block_comment(str)
 43 
 44 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 45                                                             Register addr, Register count) {
 46   bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
 47 
 48   // With G1, don't generate the call if we statically know that the target is uninitialized.
 49   if (!dest_uninitialized) {
 50     // Is marking active?
 51     Label filtered;
 52     assert_different_registers(addr,  Z_R0_scratch);  // would be destroyed by push_frame()
 53     assert_different_registers(count, Z_R0_scratch);  // would be destroyed by push_frame()
 54     Register Rtmp1 = Z_R0_scratch;
 55     const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
 56     if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 57       __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
 58     } else {
 59       guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 60       __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
 61     }
 62     __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
 63 
 64     RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame.
 65 
 66     if (UseCompressedOops) {
 67       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), addr, count);
 68     } else {
 69       __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), addr, count);
 70     }
 71 
 72     RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers);
 73 
 74     __ bind(filtered);
 75   }
 76 }
 77 
 78 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 79                                                              Register addr, Register count, bool do_return) {
 80   address entry_point = CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry);
 81   if (!do_return) {
 82     assert_different_registers(addr,  Z_R0_scratch);  // would be destroyed by push_frame()
 83     assert_different_registers(count, Z_R0_scratch);  // would be destroyed by push_frame()
 84     RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame.
 85     __ call_VM_leaf(entry_point, addr, count);
 86     RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers);
 87   } else {
 88     // Tail call: call c and return to stub caller.
 89     __ lgr_if_needed(Z_ARG1, addr);
 90     __ lgr_if_needed(Z_ARG2, count);
 91     __ load_const(Z_R1, entry_point);
 92     __ z_br(Z_R1); // Branch without linking, callee will return to stub caller.
 93   }
 94 }
 95 
 96 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
 97                                     const Address& src, Register dst, Register tmp1, Register tmp2, Label *is_null) {
 98   bool on_oop = type == T_OBJECT || type == T_ARRAY;
 99   bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
100   bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
101   bool on_reference = on_weak || on_phantom;
102   Label done;
103   if (on_oop && on_reference && is_null == NULL) { is_null = &done; }
104   ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, is_null);
105   if (on_oop && on_reference) {
106     // Generate the G1 pre-barrier code to log the value of
107     // the referent field in an SATB buffer.
108     g1_write_barrier_pre(masm, decorators | OOP_NOT_NULL,
109                          NULL /* obj */,
110                          dst  /* pre_val */,
111                          noreg/* preserve */ ,
112                          tmp1, tmp2 /* tmp */,
113                          true /* pre_val_needed */);
114   }
115   __ bind(done);
116 }
117 
118 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators,
119                                                  const Address*  obj,
120                                                  Register        Rpre_val,      // Ideally, this is a non-volatile register.
121                                                  Register        Rval,          // Will be preserved.
122                                                  Register        Rtmp1,         // If Rpre_val is volatile, either Rtmp1
123                                                  Register        Rtmp2,         // or Rtmp2 has to be non-volatile.
124                                                  bool            pre_val_needed // Save Rpre_val across runtime call, caller uses it.
125                                                  ) {
126 
127   bool not_null  = (decorators & OOP_NOT_NULL) != 0,
128        preloaded = obj == NULL;
129 
130   const Register Robj = obj ? obj->base() : noreg,
131                  Roff = obj ? obj->index() : noreg;
132   const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
133   const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
134   const int index_offset  = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
135   assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp<i> must be Z_R0!!
136   assert_different_registers(Robj, Z_R0_scratch);         // Used for addressing. Furthermore, push_frame destroys Z_R0!!
137   assert_different_registers(Rval, Z_R0_scratch);         // push_frame destroys Z_R0!!
138 
139   Label callRuntime, filtered;
140 
141   BLOCK_COMMENT("g1_write_barrier_pre {");
142 
143   // Is marking active?
144   // Note: value is loaded for test purposes only. No further use here.
145   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
146     __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
147   } else {
148     guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
149     __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
150   }
151   __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
152 
153   assert(Rpre_val != noreg, "must have a real register");
154 
155 
156   // If an object is given, we need to load the previous value into Rpre_val.
157   if (obj) {
158     // Load the previous value...
159     if (UseCompressedOops) {
160       __ z_llgf(Rpre_val, *obj);
161     } else {
162       __ z_lg(Rpre_val, *obj);
163     }
164   }
165 
166   // Is the previous value NULL?
167   // If so, we don't need to record it and we're done.
168   // Note: pre_val is loaded, decompressed and stored (directly or via runtime call).
169   //       Register contents is preserved across runtime call if caller requests to do so.
170   if (preloaded && not_null) {
171 #ifdef ASSERT
172     __ z_ltgr(Rpre_val, Rpre_val);
173     __ asm_assert_ne("null oop not allowed (G1 pre)", 0x321); // Checked by caller.
174 #endif
175   } else {
176     __ z_ltgr(Rpre_val, Rpre_val);
177     __ z_bre(filtered); // previous value is NULL, so we don't need to record it.
178   }
179 
180   // Decode the oop now. We know it's not NULL.
181   if (Robj != noreg && UseCompressedOops) {
182     __ oop_decoder(Rpre_val, Rpre_val, /*maybeNULL=*/false);
183   }
184 
185   // OK, it's not filtered, so we'll need to call enqueue.
186 
187   // We can store the original value in the thread's buffer
188   // only if index > 0. Otherwise, we need runtime to handle.
189   // (The index field is typed as size_t.)
190   Register Rbuffer = Rtmp1, Rindex = Rtmp2;
191   assert_different_registers(Rbuffer, Rindex, Rpre_val);
192 
193   __ z_lg(Rbuffer, buffer_offset, Z_thread);
194 
195   __ load_and_test_long(Rindex, Address(Z_thread, index_offset));
196   __ z_bre(callRuntime); // If index == 0, goto runtime.
197 
198   __ add2reg(Rindex, -wordSize); // Decrement index.
199   __ z_stg(Rindex, index_offset, Z_thread);
200 
201   // Record the previous value.
202   __ z_stg(Rpre_val, 0, Rbuffer, Rindex);
203   __ z_bru(filtered);  // We are done.
204 
205   Rbuffer = noreg;  // end of life
206   Rindex  = noreg;  // end of life
207 
208   __ bind(callRuntime);
209 
210   // Save some registers (inputs and result) over runtime call
211   // by spilling them into the top frame.
212   if (Robj != noreg && Robj->is_volatile()) {
213     __ z_stg(Robj, Robj->encoding()*BytesPerWord, Z_SP);
214   }
215   if (Roff != noreg && Roff->is_volatile()) {
216     __ z_stg(Roff, Roff->encoding()*BytesPerWord, Z_SP);
217   }
218   if (Rval != noreg && Rval->is_volatile()) {
219     __ z_stg(Rval, Rval->encoding()*BytesPerWord, Z_SP);
220   }
221 
222   // Save Rpre_val (result) over runtime call.
223   Register Rpre_save = Rpre_val;
224   if ((Rpre_val == Z_R0_scratch) || (pre_val_needed && Rpre_val->is_volatile())) {
225     guarantee(!Rtmp1->is_volatile() || !Rtmp2->is_volatile(), "oops!");
226     Rpre_save = !Rtmp1->is_volatile() ? Rtmp1 : Rtmp2;
227   }
228   __ lgr_if_needed(Rpre_save, Rpre_val);
229 
230   // Push frame to protect top frame with return pc and spilled register values.
231   __ save_return_pc();
232   __ push_frame_abi160(0); // Will use Z_R0 as tmp.
233 
234   // Rpre_val may be destroyed by push_frame().
235   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), Rpre_save, Z_thread);
236 
237   __ pop_frame();
238   __ restore_return_pc();
239 
240   // Restore spilled values.
241   if (Robj != noreg && Robj->is_volatile()) {
242     __ z_lg(Robj, Robj->encoding()*BytesPerWord, Z_SP);
243   }
244   if (Roff != noreg && Roff->is_volatile()) {
245     __ z_lg(Roff, Roff->encoding()*BytesPerWord, Z_SP);
246   }
247   if (Rval != noreg && Rval->is_volatile()) {
248     __ z_lg(Rval, Rval->encoding()*BytesPerWord, Z_SP);
249   }
250   if (pre_val_needed && Rpre_val->is_volatile()) {
251     __ lgr_if_needed(Rpre_val, Rpre_save);
252   }
253 
254   __ bind(filtered);
255   BLOCK_COMMENT("} g1_write_barrier_pre");
256 }
257 
258 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, Register Rstore_addr, Register Rnew_val,
259                                                   Register Rtmp1, Register Rtmp2, Register Rtmp3) {
260   bool not_null = (decorators & OOP_NOT_NULL) != 0;
261 
262   assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); // Most probably, Rnew_val == Rtmp3.
263 
264   Label callRuntime, filtered;
265 
266   CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
267   assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
268 
269   BLOCK_COMMENT("g1_write_barrier_post {");
270 
271   // Does store cross heap regions?
272   // It does if the two addresses specify different grain addresses.
273   if (G1RSBarrierRegionFilter) {
274     if (VM_Version::has_DistinctOpnds()) {
275       __ z_xgrk(Rtmp1, Rstore_addr, Rnew_val);
276     } else {
277       __ z_lgr(Rtmp1, Rstore_addr);
278       __ z_xgr(Rtmp1, Rnew_val);
279     }
280     __ z_srag(Rtmp1, Rtmp1, HeapRegion::LogOfHRGrainBytes);
281     __ z_bre(filtered);
282   }
283 
284   // Crosses regions, storing NULL?
285   if (not_null) {
286 #ifdef ASSERT
287     __ z_ltgr(Rnew_val, Rnew_val);
288     __ asm_assert_ne("null oop not allowed (G1 post)", 0x322); // Checked by caller.
289 #endif
290   } else {
291     __ z_ltgr(Rnew_val, Rnew_val);
292     __ z_bre(filtered);
293   }
294 
295   Rnew_val = noreg; // end of lifetime
296 
297   // Storing region crossing non-NULL, is card already dirty?
298   assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
299   assert_different_registers(Rtmp1, Rtmp2, Rtmp3);
300   // Make sure not to use Z_R0 for any of these registers.
301   Register Rcard_addr = (Rtmp1 != Z_R0_scratch) ? Rtmp1 : Rtmp3;
302   Register Rbase      = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp3;
303 
304   // calculate address of card
305   __ load_const_optimized(Rbase, (address)ct->card_table()->byte_map_base());      // Card table base.
306   __ z_srlg(Rcard_addr, Rstore_addr, CardTable::card_shift);         // Index into card table.
307   __ z_algr(Rcard_addr, Rbase);                                      // Explicit calculation needed for cli.
308   Rbase = noreg; // end of lifetime
309 
310   // Filter young.
311   assert((unsigned int)G1CardTable::g1_young_card_val() <= 255, "otherwise check this code");
312   __ z_cli(0, Rcard_addr, G1CardTable::g1_young_card_val());
313   __ z_bre(filtered);
314 
315   // Check the card value. If dirty, we're done.
316   // This also avoids false sharing of the (already dirty) card.
317   __ z_sync(); // Required to support concurrent cleaning.
318   assert((unsigned int)G1CardTable::dirty_card_val() <= 255, "otherwise check this code");
319   __ z_cli(0, Rcard_addr, G1CardTable::dirty_card_val()); // Reload after membar.
320   __ z_bre(filtered);
321 
322   // Storing a region crossing, non-NULL oop, card is clean.
323   // Dirty card and log.
324   __ z_mvi(0, Rcard_addr, G1CardTable::dirty_card_val());
325 
326   Register Rcard_addr_x = Rcard_addr;
327   Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
328   Register Rqueue_buf   = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1;
329   const int qidx_off    = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
330   const int qbuf_off    = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
331   if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) {
332     Rcard_addr_x = Z_R0_scratch;  // Register shortage. We have to use Z_R0.
333   }
334   __ lgr_if_needed(Rcard_addr_x, Rcard_addr);
335 
336   __ load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off));
337   __ z_bre(callRuntime); // Index == 0 then jump to runtime.
338 
339   __ z_lg(Rqueue_buf, qbuf_off, Z_thread);
340 
341   __ add2reg(Rqueue_index, -wordSize); // Decrement index.
342   __ z_stg(Rqueue_index, qidx_off, Z_thread);
343 
344   __ z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card.
345   __ z_bru(filtered);
346 
347   __ bind(callRuntime);
348 
349   // TODO: do we need a frame? Introduced to be on the safe side.
350   bool needs_frame = true;
351   __ lgr_if_needed(Rcard_addr, Rcard_addr_x); // copy back asap. push_frame will destroy Z_R0_scratch!
352 
353   // VM call need frame to access(write) O register.
354   if (needs_frame) {
355     __ save_return_pc();
356     __ push_frame_abi160(0); // Will use Z_R0 as tmp on old CPUs.
357   }
358 
359   // Save the live input values.
360   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, Z_thread);
361 
362   if (needs_frame) {
363     __ pop_frame();
364     __ restore_return_pc();
365   }
366 
367   __ bind(filtered);
368 
369   BLOCK_COMMENT("} g1_write_barrier_post");
370 }
371 
372 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
373                                          const Address& dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
374   bool on_array = (decorators & IN_HEAP_ARRAY) != 0;
375   bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
376   bool precise = on_array || on_anonymous;
377   // Load and record the previous value.
378   g1_write_barrier_pre(masm, decorators, &dst, tmp3, val, tmp1, tmp2, false);
379 
380   BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
381 
382   // No need for post barrier if storing NULL
383   if (val != noreg) {
384     const Register base = dst.base(),
385                    idx  = dst.index();
386     const intptr_t disp = dst.disp();
387     if (precise && (disp != 0 || idx != noreg)) {
388       __ add2reg_with_index(base, disp, idx, base);
389     }
390     g1_write_barrier_post(masm, decorators, base, val, tmp1, tmp2, tmp3);
391   }
392 }
393 
394 void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2) {
395   NearLabel Ldone, Lnot_weak;
396   __ z_ltgr(tmp1, value);
397   __ z_bre(Ldone);          // Use NULL result as-is.
398 
399   __ z_nill(value, ~JNIHandles::weak_tag_mask);
400   __ z_lg(value, 0, value); // Resolve (untagged) jobject.
401 
402   __ z_tmll(tmp1, JNIHandles::weak_tag_mask); // Test for jweak tag.
403   __ z_braz(Lnot_weak);
404   __ verify_oop(value);
405   DecoratorSet decorators = IN_ROOT | ON_PHANTOM_OOP_REF;
406   g1_write_barrier_pre(masm, decorators, (const Address*)NULL, value, noreg, tmp1, tmp2, true);
407   __ bind(Lnot_weak);
408   __ verify_oop(value);
409   __ bind(Ldone);
410 }
411 
412 #ifdef COMPILER1
413 
414 #undef __
415 #define __ ce->masm()->
416 
417 void G1BarrierSetAssembler::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
418   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
419   // At this point we know that marking is in progress.
420   // If do_load() is true then we have to emit the
421   // load of the previous value; otherwise it has already
422   // been loaded into _pre_val.
423   __ bind(*stub->entry());
424   ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots.
425   assert(stub->pre_val()->is_register(), "Precondition.");
426 
427   Register pre_val_reg = stub->pre_val()->as_register();
428 
429   if (stub->do_load()) {
430     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
431   }
432 
433   __ z_ltgr(Z_R1_scratch, pre_val_reg); // Pass oop in Z_R1_scratch to Runtime1::g1_pre_barrier_slow_id.
434   __ branch_optimized(Assembler::bcondZero, *stub->continuation());
435   ce->emit_call_c(bs->pre_barrier_c1_runtime_code_blob()->code_begin());
436   __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
437 }
438 
439 void G1BarrierSetAssembler::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
440   G1BarrierSetC1* bs = (G1BarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
441   __ bind(*stub->entry());
442   ce->check_reserved_argument_area(16); // RT stub needs 2 spill slots.
443   assert(stub->addr()->is_register(), "Precondition.");
444   assert(stub->new_val()->is_register(), "Precondition.");
445   Register new_val_reg = stub->new_val()->as_register();
446   __ z_ltgr(new_val_reg, new_val_reg);
447   __ branch_optimized(Assembler::bcondZero, *stub->continuation());
448   __ z_lgr(Z_R1_scratch, stub->addr()->as_pointer_register());
449   ce->emit_call_c(bs->post_barrier_c1_runtime_code_blob()->code_begin());
450   __ branch_optimized(Assembler::bcondAlways, *stub->continuation());
451 }
452 
453 #undef __
454 
455 #define __ sasm->
456 
457 static OopMap* save_volatile_registers(StubAssembler* sasm, Register return_pc = Z_R14) {
458   __ block_comment("save_volatile_registers");
459   RegisterSaver::RegisterSet reg_set = RegisterSaver::all_volatile_registers;
460   int frame_size_in_slots = RegisterSaver::live_reg_frame_size(reg_set) / VMRegImpl::stack_slot_size;
461   sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word);
462   return RegisterSaver::save_live_registers(sasm, reg_set, return_pc);
463 }
464 
465 static void restore_volatile_registers(StubAssembler* sasm) {
466   __ block_comment("restore_volatile_registers");
467   RegisterSaver::RegisterSet reg_set = RegisterSaver::all_volatile_registers;
468   RegisterSaver::restore_live_registers(sasm, reg_set);
469 }
470 
471 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
472   // Z_R1_scratch: previous value of memory
473 
474   BarrierSet* bs = BarrierSet::barrier_set();
475   __ set_info("g1_pre_barrier_slow_id", false);
476 
477   Register pre_val = Z_R1_scratch;
478   Register tmp  = Z_R6; // Must be non-volatile because it is used to save pre_val.
479   Register tmp2 = Z_R7;
480 
481   Label refill, restart, marking_not_active;
482   int satb_q_active_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
483   int satb_q_index_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
484   int satb_q_buf_byte_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
485 
486   // Save tmp registers (see assertion in G1PreBarrierStub::emit_code()).
487   __ z_stg(tmp,  0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
488   __ z_stg(tmp2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
489 
490   // Is marking still active?
491   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
492     __ load_and_test_int(tmp, Address(Z_thread, satb_q_active_byte_offset));
493   } else {
494     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
495     __ load_and_test_byte(tmp, Address(Z_thread, satb_q_active_byte_offset));
496   }
497   __ z_bre(marking_not_active); // Activity indicator is zero, so there is no marking going on currently.
498 
499   __ bind(restart);
500   // Load the index into the SATB buffer. SATBMarkQueue::_index is a
501   // size_t so ld_ptr is appropriate.
502   __ z_ltg(tmp, satb_q_index_byte_offset, Z_R0, Z_thread);
503 
504   // index == 0?
505   __ z_brz(refill);
506 
507   __ z_lg(tmp2, satb_q_buf_byte_offset, Z_thread);
508   __ add2reg(tmp, -oopSize);
509 
510   __ z_stg(pre_val, 0, tmp, tmp2); // [_buf + index] := <address_of_card>
511   __ z_stg(tmp, satb_q_index_byte_offset, Z_thread);
512 
513   __ bind(marking_not_active);
514   // Restore tmp registers (see assertion in G1PreBarrierStub::emit_code()).
515   __ z_lg(tmp,  0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
516   __ z_lg(tmp2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
517   __ z_br(Z_R14);
518 
519   __ bind(refill);
520   save_volatile_registers(sasm);
521   __ z_lgr(tmp, pre_val); // save pre_val
522   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SATBMarkQueueSet::handle_zero_index_for_thread),
523                   Z_thread);
524   __ z_lgr(pre_val, tmp); // restore pre_val
525   restore_volatile_registers(sasm);
526   __ z_bru(restart);
527 }
528 
529 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
530   // Z_R1_scratch: oop address, address of updated memory slot
531 
532   BarrierSet* bs = BarrierSet::barrier_set();
533   __ set_info("g1_post_barrier_slow_id", false);
534 
535   Register addr_oop  = Z_R1_scratch;
536   Register addr_card = Z_R1_scratch;
537   Register r1        = Z_R6; // Must be saved/restored.
538   Register r2        = Z_R7; // Must be saved/restored.
539   Register cardtable = r1;   // Must be non-volatile, because it is used to save addr_card.
540   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
541   CardTable* ct = ctbs->card_table();
542   jbyte* byte_map_base = ct->byte_map_base();
543 
544   // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
545   __ z_stg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
546 
547   Label not_already_dirty, restart, refill, young_card;
548 
549   // Calculate address of card corresponding to the updated oop slot.
550   AddressLiteral rs(byte_map_base);
551   __ z_srlg(addr_card, addr_oop, CardTable::card_shift);
552   addr_oop = noreg; // dead now
553   __ load_const_optimized(cardtable, rs); // cardtable := <card table base>
554   __ z_agr(addr_card, cardtable); // addr_card := addr_oop>>card_shift + cardtable
555 
556   __ z_cli(0, addr_card, (int)G1CardTable::g1_young_card_val());
557   __ z_bre(young_card);
558 
559   __ z_sync(); // Required to support concurrent cleaning.
560 
561   __ z_cli(0, addr_card, (int)CardTable::dirty_card_val());
562   __ z_brne(not_already_dirty);
563 
564   __ bind(young_card);
565   // We didn't take the branch, so we're already dirty: restore
566   // used registers and return.
567   __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
568   __ z_br(Z_R14);
569 
570   // Not dirty.
571   __ bind(not_already_dirty);
572 
573   // First, dirty it: [addr_card] := 0
574   __ z_mvi(0, addr_card, CardTable::dirty_card_val());
575 
576   Register idx = cardtable; // Must be non-volatile, because it is used to save addr_card.
577   Register buf = r2;
578   cardtable = noreg; // now dead
579 
580   // Save registers used below (see assertion in G1PreBarrierStub::emit_code()).
581   __ z_stg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
582 
583   ByteSize dirty_card_q_index_byte_offset = G1ThreadLocalData::dirty_card_queue_index_offset();
584   ByteSize dirty_card_q_buf_byte_offset = G1ThreadLocalData::dirty_card_queue_buffer_offset();
585 
586   __ bind(restart);
587 
588   // Get the index into the update buffer. DirtyCardQueue::_index is
589   // a size_t so z_ltg is appropriate here.
590   __ z_ltg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
591 
592   // index == 0?
593   __ z_brz(refill);
594 
595   __ z_lg(buf, Address(Z_thread, dirty_card_q_buf_byte_offset));
596   __ add2reg(idx, -oopSize);
597 
598   __ z_stg(addr_card, 0, idx, buf); // [_buf + index] := <address_of_card>
599   __ z_stg(idx, Address(Z_thread, dirty_card_q_index_byte_offset));
600   // Restore killed registers and return.
601   __ z_lg(r1, 0*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
602   __ z_lg(r2, 1*BytesPerWord + FrameMap::first_available_sp_in_frame, Z_SP);
603   __ z_br(Z_R14);
604 
605   __ bind(refill);
606   save_volatile_registers(sasm);
607   __ z_lgr(idx, addr_card); // Save addr_card, tmp3 must be non-volatile.
608   __ call_VM_leaf(CAST_FROM_FN_PTR(address, DirtyCardQueueSet::handle_zero_index_for_thread),
609                                    Z_thread);
610   __ z_lgr(addr_card, idx);
611   restore_volatile_registers(sasm); // Restore addr_card.
612   __ z_bru(restart);
613 }
614 
615 #undef __
616 
617 #endif // COMPILER1