1 /*
  2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/macroAssembler.inline.hpp"
 27 #include "gc/g1/g1BarrierSet.hpp"
 28 #include "gc/g1/g1BarrierSetAssembler.hpp"
 29 #include "gc/g1/g1CardTable.hpp"
 30 #include "gc/g1/heapRegion.hpp"
 31 #include "interpreter/interp_masm.hpp"
 32 #include "runtime/sharedRuntime.hpp"
 33 #include "runtime/thread.hpp"
 34 #include "utilities/macros.hpp"
 35 
 36 #define __ masm->
 37 
 38 #ifdef PRODUCT
 39 #define BLOCK_COMMENT(str) /* nothing */
 40 #else
 41 #define BLOCK_COMMENT(str) __ block_comment(str)
 42 #endif
 43 
 44 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 45 
 46 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
 47                                                             Register addr, Register count, int callee_saved_regs) {
 48   bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
 49   if (!dest_uninitialized) {
 50     assert( addr->encoding() < callee_saved_regs, "addr must be saved");
 51     assert(count->encoding() < callee_saved_regs, "count must be saved");
 52 
 53     BLOCK_COMMENT("PreBarrier");
 54 
 55 #ifdef AARCH64
 56     callee_saved_regs = align_up(callee_saved_regs, 2);
 57     for (int i = 0; i < callee_saved_regs; i += 2) {
 58       __ raw_push(as_Register(i), as_Register(i+1));
 59     }
 60 #else
 61     RegisterSet saved_regs = RegisterSet(R0, as_Register(callee_saved_regs-1));
 62     __ push(saved_regs | R9ifScratched);
 63 #endif // AARCH64
 64 
 65     if (addr != R0) {
 66       assert_different_registers(count, R0);
 67       __ mov(R0, addr);
 68     }
 69 #ifdef AARCH64
 70     __ zero_extend(R1, count, 32); // G1BarrierSet::write_ref_array_pre_*_entry takes size_t
 71 #else
 72     if (count != R1) {
 73       __ mov(R1, count);
 74     }
 75 #endif // AARCH64
 76 
 77     if (UseCompressedOops) {
 78       __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry));
 79     } else {
 80       __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry));
 81     }
 82 
 83 #ifdef AARCH64
 84     for (int i = callee_saved_regs - 2; i >= 0; i -= 2) {
 85       __ raw_pop(as_Register(i), as_Register(i+1));
 86     }
 87 #else
 88     __ pop(saved_regs | R9ifScratched);
 89 #endif // AARCH64
 90   }
 91 }
 92 
 93 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
 94                                                              Register addr, Register count, Register tmp) {
 95 
 96   BLOCK_COMMENT("G1PostBarrier");
 97   if (addr != R0) {
 98     assert_different_registers(count, R0);
 99     __ mov(R0, addr);
100   }
101 #ifdef AARCH64
102   __ zero_extend(R1, count, 32); // G1BarrierSet::write_ref_array_post_entry takes size_t
103 #else
104   if (count != R1) {
105     __ mov(R1, count);
106   }
107 #if R9_IS_SCRATCHED
108   // Safer to save R9 here since callers may have been written
109   // assuming R9 survives. This is suboptimal but is not in
110   // general worth optimizing for the few platforms where R9
111   // is scratched. Note that the optimization might not be to
112   // difficult for this particular call site.
113   __ push(R9);
114 #endif // !R9_IS_SCRATCHED
115 #endif // !AARCH64
116   __ call(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry));
117 #ifndef AARCH64
118 #if R9_IS_SCRATCHED
119   __ pop(R9);
120 #endif // !R9_IS_SCRATCHED
121 #endif // !AARCH64
122 }
123 
124 #ifdef COMPILER1
125 
126 #undef __
127 #define __ ce->masm()->
128 
129 void G1BarrierSetAssembler::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
130   // At this point we know that marking is in progress.
131   // If do_load() is true then we have to emit the
132   // load of the previous value; otherwise it has already
133   // been loaded into _pre_val.
134 
135   __ bind(*stub->entry());
136   assert(stub->pre_val()->is_register(), "Precondition.");
137 
138   Register pre_val_reg = stub->pre_val()->as_register();
139 
140   if (stub->do_load()) {
141     ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
142   }
143 
144   __ cbz(pre_val_reg, *stub->continuation());
145   ce->verify_reserved_argument_area_size(1);
146   __ str(pre_val_reg, Address(SP));
147   __ call(Runtime1::entry_for(Runtime1::g1_pre_barrier_slow_id), relocInfo::runtime_call_type);
148 
149   __ b(*stub->continuation());
150 }
151 
152 void G1BarrierSetAssembler::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
153   __ bind(*stub->entry());
154   assert(stub->addr()->is_register(), "Precondition.");
155   assert(stub->new_val()->is_register(), "Precondition.");
156   Register new_val_reg = stub->new_val()->as_register();
157   __ cbz(new_val_reg, *stub->continuation());
158   ce->verify_reserved_argument_area_size(1);
159   __ str(stub->addr()->as_pointer_register(), Address(SP));
160   __ call(Runtime1::entry_for(Runtime1::g1_post_barrier_slow_id), relocInfo::runtime_call_type);
161   __ b(*stub->continuation());
162 }
163 
164 #undef __
165 #define __ sasm->
166 
167 void G1BarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
168   // Input:
169   // - pre_val pushed on the stack
170 
171   __ set_info("g1_pre_barrier_slow_id", dont_gc_arguments);
172 
173   BarrierSet* bs = BarrierSet::barrier_set();
174   if (bs->kind() != BarrierSet::G1BarrierSet) {
175     __ mov(R0, (int)id);
176     __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), R0);
177     __ should_not_reach_here();
178     break;
179   }
180 
181   // save at least the registers that need saving if the runtime is called
182 #ifdef AARCH64
183   __ raw_push(R0, R1);
184   __ raw_push(R2, R3);
185   const int nb_saved_regs = 4;
186 #else // AARCH64
187   const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);
188   const int nb_saved_regs = 6;
189   assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");
190   __ push(saved_regs);
191 #endif // AARCH64
192 
193   const Register r_pre_val_0  = R0; // must be R0, to be ready for the runtime call
194   const Register r_index_1    = R1;
195   const Register r_buffer_2   = R2;
196 
197   Address queue_active(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
198   Address queue_index(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
199   Address buffer(Rthread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
200 
201   Label done;
202   Label runtime;
203 
204   // Is marking still active?
205   assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
206   __ ldrb(R1, queue_active);
207   __ cbz(R1, done);
208 
209   __ ldr(r_index_1, queue_index);
210   __ ldr(r_pre_val_0, Address(SP, nb_saved_regs*wordSize));
211   __ ldr(r_buffer_2, buffer);
212 
213   __ subs(r_index_1, r_index_1, wordSize);
214   __ b(runtime, lt);
215 
216   __ str(r_index_1, queue_index);
217   __ str(r_pre_val_0, Address(r_buffer_2, r_index_1));
218 
219   __ bind(done);
220 
221 #ifdef AARCH64
222   __ raw_pop(R2, R3);
223   __ raw_pop(R0, R1);
224 #else // AARCH64
225   __ pop(saved_regs);
226 #endif // AARCH64
227 
228   __ ret();
229 
230   __ bind(runtime);
231 
232   __ save_live_registers();
233 
234   assert(r_pre_val_0 == c_rarg0, "pre_val should be in R0");
235   __ mov(c_rarg1, Rthread);
236   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), c_rarg0, c_rarg1);
237 
238   __ restore_live_registers_without_return();
239 
240   __ b(done);
241 }
242 
243 void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
244   // Input:
245   // - store_addr, pushed on the stack
246 
247   __ set_info("g1_post_barrier_slow_id", dont_gc_arguments);
248 
249   BarrierSet* bs = BarrierSet::barrier_set();
250   if (bs->kind() != BarrierSet::G1BarrierSet) {
251     __ mov(R0, (int)id);
252     __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), R0);
253     __ should_not_reach_here();
254     break;
255   }
256 
257   Label done;
258   Label recheck;
259   Label runtime;
260 
261   Address queue_index(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
262   Address buffer(Rthread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
263 
264   AddressLiteral cardtable(ci_card_table_address_as<address>(), relocInfo::none);
265 
266   // save at least the registers that need saving if the runtime is called
267 #ifdef AARCH64
268   __ raw_push(R0, R1);
269   __ raw_push(R2, R3);
270   const int nb_saved_regs = 4;
271 #else // AARCH64
272   const RegisterSet saved_regs = RegisterSet(R0,R3) | RegisterSet(R12) | RegisterSet(LR);
273   const int nb_saved_regs = 6;
274   assert(nb_saved_regs == saved_regs.size(), "fix nb_saved_regs");
275   __ push(saved_regs);
276 #endif // AARCH64
277 
278   const Register r_card_addr_0 = R0; // must be R0 for the slow case
279   const Register r_obj_0 = R0;
280   const Register r_card_base_1 = R1;
281   const Register r_tmp2 = R2;
282   const Register r_index_2 = R2;
283   const Register r_buffer_3 = R3;
284   const Register tmp1 = Rtemp;
285 
286   __ ldr(r_obj_0, Address(SP, nb_saved_regs*wordSize));
287   // Note: there is a comment in x86 code about not using
288   // ExternalAddress / lea, due to relocation not working
289   // properly for that address. Should be OK for arm, where we
290   // explicitly specify that 'cardtable' has a relocInfo::none
291   // type.
292   __ lea(r_card_base_1, cardtable);
293   __ add(r_card_addr_0, r_card_base_1, AsmOperand(r_obj_0, lsr, CardTable::card_shift));
294 
295   // first quick check without barrier
296   __ ldrb(r_tmp2, Address(r_card_addr_0));
297 
298   __ cmp(r_tmp2, (int)G1CardTable::g1_young_card_val());
299   __ b(recheck, ne);
300 
301   __ bind(done);
302 
303 #ifdef AARCH64
304   __ raw_pop(R2, R3);
305   __ raw_pop(R0, R1);
306 #else // AARCH64
307   __ pop(saved_regs);
308 #endif // AARCH64
309 
310   __ ret();
311 
312   __ bind(recheck);
313 
314   __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), tmp1);
315 
316   // reload card state after the barrier that ensures the stored oop was visible
317   __ ldrb(r_tmp2, Address(r_card_addr_0));
318 
319   assert(CardTable::dirty_card_val() == 0, "adjust this code");
320   __ cbz(r_tmp2, done);
321 
322   // storing region crossing non-NULL, card is clean.
323   // dirty card and log.
324 
325   assert(0 == (int)CardTable::dirty_card_val(), "adjust this code");
326   if ((ci_card_table_address_as<intptr_t>() & 0xff) == 0) {
327     // Card table is aligned so the lowest byte of the table address base is zero.
328     __ strb(r_card_base_1, Address(r_card_addr_0));
329   } else {
330     __ strb(__ zero_register(r_tmp2), Address(r_card_addr_0));
331   }
332 
333   __ ldr(r_index_2, queue_index);
334   __ ldr(r_buffer_3, buffer);
335 
336   __ subs(r_index_2, r_index_2, wordSize);
337   __ b(runtime, lt); // go to runtime if now negative
338 
339   __ str(r_index_2, queue_index);
340 
341   __ str(r_card_addr_0, Address(r_buffer_3, r_index_2));
342 
343   __ b(done);
344 
345   __ bind(runtime);
346 
347   __ save_live_registers();
348 
349   assert(r_card_addr_0 == c_rarg0, "card_addr should be in R0");
350   __ mov(c_rarg1, Rthread);
351   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), c_rarg0, c_rarg1);
352 
353   __ restore_live_registers_without_return();
354 
355   __ b(done);
356 }
357 
358 #undef __
359 
360 #endif // COMPILER1