1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "gc/g1/g1BarrierSet.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1BarrierSetAssembler.hpp"
31 #include "gc/g1/g1ThreadLocalData.hpp"
32 #include "gc/g1/heapRegion.hpp"
33 #include "interpreter/interp_masm.hpp"
34 #include "runtime/sharedRuntime.hpp"
35
36 #define __ masm->
37
38 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
39 Register from, Register to, Register count,
40 Register preserve1, Register preserve2) {
41 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
42 // With G1, don't generate the call if we statically know that the target in uninitialized
43 if (!dest_uninitialized) {
44 int spill_slots = 3;
45 if (preserve1 != noreg) { spill_slots++; }
46 if (preserve2 != noreg) { spill_slots++; }
47 const int frame_size = align_up(frame::abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes);
48 Label filtered;
49
50 // Is marking active?
51 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
52 __ lwz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
53 } else {
54 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
55 __ lbz(R0, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
56 }
57 __ cmpdi(CCR0, R0, 0);
58 __ beq(CCR0, filtered);
59
60 __ save_LR_CR(R0);
61 __ push_frame(frame_size, R0);
62 int slot_nr = 0;
63 __ std(from, frame_size - (++slot_nr) * wordSize, R1_SP);
64 __ std(to, frame_size - (++slot_nr) * wordSize, R1_SP);
65 __ std(count, frame_size - (++slot_nr) * wordSize, R1_SP);
66 if (preserve1 != noreg) { __ std(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); }
67 if (preserve2 != noreg) { __ std(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); }
68
69 if (UseCompressedOops) {
70 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), to, count);
71 } else {
72 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), to, count);
73 }
74
75 slot_nr = 0;
76 __ ld(from, frame_size - (++slot_nr) * wordSize, R1_SP);
77 __ ld(to, frame_size - (++slot_nr) * wordSize, R1_SP);
78 __ ld(count, frame_size - (++slot_nr) * wordSize, R1_SP);
79 if (preserve1 != noreg) { __ ld(preserve1, frame_size - (++slot_nr) * wordSize, R1_SP); }
80 if (preserve2 != noreg) { __ ld(preserve2, frame_size - (++slot_nr) * wordSize, R1_SP); }
81 __ addi(R1_SP, R1_SP, frame_size); // pop_frame()
82 __ restore_LR_CR(R0);
83
84 __ bind(filtered);
85 }
86 }
87
88 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
89 Register addr, Register count, Register preserve) {
90 int spill_slots = (preserve != noreg) ? 1 : 0;
91 const int frame_size = align_up(frame::abi_reg_args_size + spill_slots * BytesPerWord, frame::alignment_in_bytes);
92
93 __ save_LR_CR(R0);
94 __ push_frame(frame_size, R0);
95 if (preserve != noreg) { __ std(preserve, frame_size - 1 * wordSize, R1_SP); }
96 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry), addr, count);
97 if (preserve != noreg) { __ ld(preserve, frame_size - 1 * wordSize, R1_SP); }
98 __ addi(R1_SP, R1_SP, frame_size); // pop_frame();
99 __ restore_LR_CR(R0);
100 }
101
102 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators, Register obj, RegisterOrConstant ind_or_offs, Register pre_val,
103 Register tmp1, Register tmp2, bool needs_frame) {
104 bool not_null = (decorators & OOP_NOT_NULL) != 0,
105 preloaded = obj == noreg;
106 Register nv_save = noreg;
107
108 if (preloaded) {
109 // We are not loading the previous value so make
110 // sure that we don't trash the value in pre_val
111 // with the code below.
112 assert_different_registers(pre_val, tmp1, tmp2);
113 if (pre_val->is_volatile()) {
114 nv_save = !tmp1->is_volatile() ? tmp1 : tmp2;
115 assert(!nv_save->is_volatile(), "need one nv temp register if pre_val lives in volatile register");
116 }
117 }
118
119 Label runtime, filtered;
120
121 // Is marking active?
122 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
123 __ lwz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
124 } else {
125 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
126 __ lbz(tmp1, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()), R16_thread);
127 }
128 __ cmpdi(CCR0, tmp1, 0);
129 __ beq(CCR0, filtered);
130
131 // Do we need to load the previous value?
132 if (!preloaded) {
133 // Load the previous value...
134 if (UseCompressedOops) {
135 __ lwz(pre_val, ind_or_offs, obj);
136 } else {
137 __ ld(pre_val, ind_or_offs, obj);
138 }
139 // Previous value has been loaded into Rpre_val.
140 }
141 assert(pre_val != noreg, "must have a real register");
142
143 // Is the previous value null?
144 if (preloaded && not_null) {
145 #ifdef ASSERT
146 __ cmpdi(CCR0, pre_val, 0);
147 __ asm_assert_ne("null oop not allowed (G1 pre)", 0x321); // Checked by caller.
148 #endif
149 } else {
150 __ cmpdi(CCR0, pre_val, 0);
151 __ beq(CCR0, filtered);
152 }
153
154 if (!preloaded && UseCompressedOops) {
155 __ decode_heap_oop_not_null(pre_val);
156 }
157
158 // OK, it's not filtered, so we'll need to call enqueue. In the normal
159 // case, pre_val will be a scratch G-reg, but there are some cases in
160 // which it's an O-reg. In the first case, do a normal call. In the
161 // latter, do a save here and call the frameless version.
162
163 // Can we store original value in the thread's buffer?
164 // Is index == 0?
165 // (The index field is typed as size_t.)
166 const Register Rbuffer = tmp1, Rindex = tmp2;
167
168 __ ld(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
169 __ cmpdi(CCR0, Rindex, 0);
170 __ beq(CCR0, runtime); // If index == 0, goto runtime.
171 __ ld(Rbuffer, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()), R16_thread);
172
173 __ addi(Rindex, Rindex, -wordSize); // Decrement index.
174 __ std(Rindex, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()), R16_thread);
175
176 // Record the previous value.
177 __ stdx(pre_val, Rbuffer, Rindex);
178 __ b(filtered);
179
180 __ bind(runtime);
181
182 // May need to preserve LR. Also needed if current frame is not compatible with C calling convention.
183 if (needs_frame) {
184 __ save_LR_CR(tmp1);
185 __ push_frame_reg_args(0, tmp2);
186 }
187
188 if (pre_val->is_volatile() && preloaded) { __ mr(nv_save, pre_val); } // Save pre_val across C call if it was preloaded.
189 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, R16_thread);
190 if (pre_val->is_volatile() && preloaded) { __ mr(pre_val, nv_save); } // restore
191
192 if (needs_frame) {
193 __ pop_frame();
194 __ restore_LR_CR(tmp1);
195 }
196
197 __ bind(filtered);
198 }
199
200 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, Register store_addr, Register new_val,
201 Register tmp1, Register tmp2, Register tmp3) {
202 bool not_null = (decorators & OOP_NOT_NULL) != 0;
203
204 Label runtime, filtered;
205 assert_different_registers(store_addr, new_val, tmp1, tmp2);
206
207 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
208 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
209
210 // Does store cross heap regions?
211 if (G1RSBarrierRegionFilter) {
212 __ xorr(tmp1, store_addr, new_val);
213 __ srdi_(tmp1, tmp1, HeapRegion::LogOfHRGrainBytes);
214 __ beq(CCR0, filtered);
215 }
216
217 // Crosses regions, storing NULL?
218 if (not_null) {
219 #ifdef ASSERT
220 __ cmpdi(CCR0, new_val, 0);
221 __ asm_assert_ne("null oop not allowed (G1 post)", 0x322); // Checked by caller.
222 #endif
223 } else {
224 __ cmpdi(CCR0, new_val, 0);
225 __ beq(CCR0, filtered);
226 }
227
228 // Storing region crossing non-NULL, is card already dirty?
229 const Register Rcard_addr = tmp1;
230 Register Rbase = tmp2;
231 __ load_const_optimized(Rbase, (address)(ct->card_table()->byte_map_base()), /*temp*/ tmp3);
232
233 __ srdi(Rcard_addr, store_addr, CardTable::card_shift);
234
235 // Get the address of the card.
236 __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr);
237 __ cmpwi(CCR0, tmp3, (int)G1CardTable::g1_young_card_val());
238 __ beq(CCR0, filtered);
239
240 __ membar(Assembler::StoreLoad);
241 __ lbzx(/*card value*/ tmp3, Rbase, Rcard_addr); // Reload after membar.
242 __ cmpwi(CCR0, tmp3 /* card value */, (int)G1CardTable::dirty_card_val());
243 __ beq(CCR0, filtered);
244
245 // Storing a region crossing, non-NULL oop, card is clean.
246 // Dirty card and log.
247 __ li(tmp3, (int)G1CardTable::dirty_card_val());
248 //release(); // G1: oops are allowed to get visible after dirty marking.
249 __ stbx(tmp3, Rbase, Rcard_addr);
250
251 __ add(Rcard_addr, Rbase, Rcard_addr); // This is the address which needs to get enqueued.
252 Rbase = noreg; // end of lifetime
253
254 const Register Rqueue_index = tmp2,
255 Rqueue_buf = tmp3;
256 __ ld(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
257 __ cmpdi(CCR0, Rqueue_index, 0);
258 __ beq(CCR0, runtime); // index == 0 then jump to runtime
259 __ ld(Rqueue_buf, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()), R16_thread);
260
261 __ addi(Rqueue_index, Rqueue_index, -wordSize); // decrement index
262 __ std(Rqueue_index, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()), R16_thread);
263
264 __ stdx(Rcard_addr, Rqueue_buf, Rqueue_index); // store card
265 __ b(filtered);
266
267 __ bind(runtime);
268
269 // Save the live input values.
270 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, R16_thread);
271
272 __ bind(filtered);
273 }
274
275 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
276 Register base, RegisterOrConstant ind_or_offs, Register val,
277 Register tmp1, Register tmp2, Register tmp3, bool needs_frame) {
278 bool on_array = (decorators & IN_HEAP_ARRAY) != 0;
279 bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
280 bool precise = on_array || on_anonymous;
281 // Load and record the previous value.
282 g1_write_barrier_pre(masm, decorators, base, ind_or_offs,
283 tmp1, tmp2, tmp3, needs_frame);
284
285 BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, needs_frame);
286
287 // No need for post barrier if storing NULL
288 if (val != noreg) {
289 if (precise) {
290 if (ind_or_offs.is_constant()) {
291 __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp1);
292 } else {
293 __ add(base, ind_or_offs.as_register(), base);
294 }
295 }
296 g1_write_barrier_post(masm, decorators, base, val, tmp1, tmp2, tmp3);
297 }
298 }
299
300 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
301 Register base, RegisterOrConstant ind_or_offs, Register dst,
302 Register tmp1, Register tmp2, bool needs_frame, Label *is_null) {
303 bool on_oop = type == T_OBJECT || type == T_ARRAY;
304 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
305 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
306 bool on_reference = on_weak || on_phantom;
307 Label done;
308 if (on_oop && on_reference && is_null == NULL) { is_null = &done; }
309 // Load the value of the referent field.
310 ModRefBarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst, tmp1, tmp2, needs_frame, is_null);
311 if (on_oop && on_reference) {
312 // Generate the G1 pre-barrier code to log the value of
313 // the referent field in an SATB buffer. Note with
314 // these parameters the pre-barrier does not generate
315 // the load of the previous value
316 // We only reach here if value is not null.
317 g1_write_barrier_pre(masm, decorators | OOP_NOT_NULL, noreg /* obj */, (intptr_t)0, dst /* pre_val */,
318 tmp1, tmp2, needs_frame);
319 }
320 __ bind(done);
321 }
322
323 void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2, bool needs_frame) {
324 Label done, not_weak;
325 __ cmpdi(CCR0, value, 0);
326 __ beq(CCR0, done); // Use NULL as-is.
327
328 __ clrrdi(tmp1, value, JNIHandles::weak_tag_size);
329 __ andi_(tmp2, value, JNIHandles::weak_tag_mask);
330 __ ld(value, 0, tmp1); // Resolve (untagged) jobject.
331
332 __ beq(CCR0, not_weak); // Test for jweak tag.
333 __ verify_oop(value);
334 g1_write_barrier_pre(masm, IN_ROOT | ON_PHANTOM_OOP_REF,
335 noreg, noreg, value,
336 tmp1, tmp2, needs_frame);
337 __ bind(not_weak);
338 __ verify_oop(value);
339 __ bind(done);
340 }
341
342 #undef __