1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "registerSaver_s390.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1BarrierSet.hpp"
31 #include "gc/g1/g1BarrierSetAssembler.hpp"
32 #include "gc/g1/g1ThreadLocalData.hpp"
33 #include "gc/g1/heapRegion.hpp"
34 #include "interpreter/interp_masm.hpp"
35 #include "runtime/sharedRuntime.hpp"
36
37 #define __ masm->
38
39 #define BLOCK_COMMENT(str) if (PrintAssembly) __ block_comment(str)
40
41 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
42 Register addr, Register count) {
43 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
44
45 // With G1, don't generate the call if we statically know that the target is uninitialized.
46 if (!dest_uninitialized) {
47 // Is marking active?
48 Label filtered;
49 assert_different_registers(addr, Z_R0_scratch); // would be destroyed by push_frame()
50 assert_different_registers(count, Z_R0_scratch); // would be destroyed by push_frame()
51 Register Rtmp1 = Z_R0_scratch;
52 const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
53 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
54 __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
55 } else {
56 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
57 __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
58 }
59 __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
60
61 RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame.
62
63 if (UseCompressedOops) {
64 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), addr, count);
65 } else {
66 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), addr, count);
67 }
68
69 RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers);
70
71 __ bind(filtered);
72 }
73 }
74
75 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
76 Register addr, Register count, bool do_return) {
77 address entry_point = CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry);
78 if (!do_return) {
79 assert_different_registers(addr, Z_R0_scratch); // would be destroyed by push_frame()
80 assert_different_registers(count, Z_R0_scratch); // would be destroyed by push_frame()
81 RegisterSaver::save_live_registers(masm, RegisterSaver::arg_registers); // Creates frame.
82 __ call_VM_leaf(entry_point, addr, count);
83 RegisterSaver::restore_live_registers(masm, RegisterSaver::arg_registers);
84 } else {
85 // Tail call: call c and return to stub caller.
86 __ lgr_if_needed(Z_ARG1, addr);
87 __ lgr_if_needed(Z_ARG2, count);
88 __ load_const(Z_R1, entry_point);
89 __ z_br(Z_R1); // Branch without linking, callee will return to stub caller.
90 }
91 }
92
93 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
94 const Address& src, Register dst, Register tmp1, Register tmp2, Label *is_null) {
95 bool on_oop = type == T_OBJECT || type == T_ARRAY;
96 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
97 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
98 bool on_reference = on_weak || on_phantom;
99 Label done;
100 if (on_oop && on_reference && is_null == NULL) { is_null = &done; }
101 ModRefBarrierSetAssembler::load_at(masm, decorators, type, src, dst, tmp1, tmp2, is_null);
102 if (on_oop && on_reference) {
103 // Generate the G1 pre-barrier code to log the value of
104 // the referent field in an SATB buffer.
105 g1_write_barrier_pre(masm, decorators | OOP_NOT_NULL,
106 NULL /* obj */,
107 dst /* pre_val */,
108 noreg/* preserve */ ,
109 tmp1, tmp2 /* tmp */,
110 true /* pre_val_needed */);
111 }
112 __ bind(done);
113 }
114
115 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm, DecoratorSet decorators,
116 const Address* obj,
117 Register Rpre_val, // Ideally, this is a non-volatile register.
118 Register Rval, // Will be preserved.
119 Register Rtmp1, // If Rpre_val is volatile, either Rtmp1
120 Register Rtmp2, // or Rtmp2 has to be non-volatile.
121 bool pre_val_needed // Save Rpre_val across runtime call, caller uses it.
122 ) {
123
124 bool not_null = (decorators & OOP_NOT_NULL) != 0,
125 preloaded = obj == NULL;
126
127 const Register Robj = obj ? obj->base() : noreg,
128 Roff = obj ? obj->index() : noreg;
129 const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
130 const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
131 const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
132 assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp<i> must be Z_R0!!
133 assert_different_registers(Robj, Z_R0_scratch); // Used for addressing. Furthermore, push_frame destroys Z_R0!!
134 assert_different_registers(Rval, Z_R0_scratch); // push_frame destroys Z_R0!!
135
136 Label callRuntime, filtered;
137
138 BLOCK_COMMENT("g1_write_barrier_pre {");
139
140 // Is marking active?
141 // Note: value is loaded for test purposes only. No further use here.
142 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
143 __ load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
144 } else {
145 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
146 __ load_and_test_byte(Rtmp1, Address(Z_thread, active_offset));
147 }
148 __ z_bre(filtered); // Activity indicator is zero, so there is no marking going on currently.
149
150 assert(Rpre_val != noreg, "must have a real register");
151
152
153 // If an object is given, we need to load the previous value into Rpre_val.
154 if (obj) {
155 // Load the previous value...
156 if (UseCompressedOops) {
157 __ z_llgf(Rpre_val, *obj);
158 } else {
159 __ z_lg(Rpre_val, *obj);
160 }
161 }
162
163 // Is the previous value NULL?
164 // If so, we don't need to record it and we're done.
165 // Note: pre_val is loaded, decompressed and stored (directly or via runtime call).
166 // Register contents is preserved across runtime call if caller requests to do so.
167 if (preloaded && not_null) {
168 #ifdef ASSERT
169 __ z_ltgr(Rpre_val, Rpre_val);
170 __ asm_assert_ne("null oop not allowed (G1 pre)", 0x321); // Checked by caller.
171 #endif
172 } else {
173 __ z_ltgr(Rpre_val, Rpre_val);
174 __ z_bre(filtered); // previous value is NULL, so we don't need to record it.
175 }
176
177 // Decode the oop now. We know it's not NULL.
178 if (Robj != noreg && UseCompressedOops) {
179 __ oop_decoder(Rpre_val, Rpre_val, /*maybeNULL=*/false);
180 }
181
182 // OK, it's not filtered, so we'll need to call enqueue.
183
184 // We can store the original value in the thread's buffer
185 // only if index > 0. Otherwise, we need runtime to handle.
186 // (The index field is typed as size_t.)
187 Register Rbuffer = Rtmp1, Rindex = Rtmp2;
188 assert_different_registers(Rbuffer, Rindex, Rpre_val);
189
190 __ z_lg(Rbuffer, buffer_offset, Z_thread);
191
192 __ load_and_test_long(Rindex, Address(Z_thread, index_offset));
193 __ z_bre(callRuntime); // If index == 0, goto runtime.
194
195 __ add2reg(Rindex, -wordSize); // Decrement index.
196 __ z_stg(Rindex, index_offset, Z_thread);
197
198 // Record the previous value.
199 __ z_stg(Rpre_val, 0, Rbuffer, Rindex);
200 __ z_bru(filtered); // We are done.
201
202 Rbuffer = noreg; // end of life
203 Rindex = noreg; // end of life
204
205 __ bind(callRuntime);
206
207 // Save some registers (inputs and result) over runtime call
208 // by spilling them into the top frame.
209 if (Robj != noreg && Robj->is_volatile()) {
210 __ z_stg(Robj, Robj->encoding()*BytesPerWord, Z_SP);
211 }
212 if (Roff != noreg && Roff->is_volatile()) {
213 __ z_stg(Roff, Roff->encoding()*BytesPerWord, Z_SP);
214 }
215 if (Rval != noreg && Rval->is_volatile()) {
216 __ z_stg(Rval, Rval->encoding()*BytesPerWord, Z_SP);
217 }
218
219 // Save Rpre_val (result) over runtime call.
220 Register Rpre_save = Rpre_val;
221 if ((Rpre_val == Z_R0_scratch) || (pre_val_needed && Rpre_val->is_volatile())) {
222 guarantee(!Rtmp1->is_volatile() || !Rtmp2->is_volatile(), "oops!");
223 Rpre_save = !Rtmp1->is_volatile() ? Rtmp1 : Rtmp2;
224 }
225 __ lgr_if_needed(Rpre_save, Rpre_val);
226
227 // Push frame to protect top frame with return pc and spilled register values.
228 __ save_return_pc();
229 __ push_frame_abi160(0); // Will use Z_R0 as tmp.
230
231 // Rpre_val may be destroyed by push_frame().
232 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), Rpre_save, Z_thread);
233
234 __ pop_frame();
235 __ restore_return_pc();
236
237 // Restore spilled values.
238 if (Robj != noreg && Robj->is_volatile()) {
239 __ z_lg(Robj, Robj->encoding()*BytesPerWord, Z_SP);
240 }
241 if (Roff != noreg && Roff->is_volatile()) {
242 __ z_lg(Roff, Roff->encoding()*BytesPerWord, Z_SP);
243 }
244 if (Rval != noreg && Rval->is_volatile()) {
245 __ z_lg(Rval, Rval->encoding()*BytesPerWord, Z_SP);
246 }
247 if (pre_val_needed && Rpre_val->is_volatile()) {
248 __ lgr_if_needed(Rpre_val, Rpre_save);
249 }
250
251 __ bind(filtered);
252 BLOCK_COMMENT("} g1_write_barrier_pre");
253 }
254
255 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm, DecoratorSet decorators, Register Rstore_addr, Register Rnew_val,
256 Register Rtmp1, Register Rtmp2, Register Rtmp3) {
257 bool not_null = (decorators & OOP_NOT_NULL) != 0;
258
259 assert_different_registers(Rstore_addr, Rnew_val, Rtmp1, Rtmp2); // Most probably, Rnew_val == Rtmp3.
260
261 Label callRuntime, filtered;
262
263 CardTableBarrierSet* ct = barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
264 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
265
266 BLOCK_COMMENT("g1_write_barrier_post {");
267
268 // Does store cross heap regions?
269 // It does if the two addresses specify different grain addresses.
270 if (G1RSBarrierRegionFilter) {
271 if (VM_Version::has_DistinctOpnds()) {
272 __ z_xgrk(Rtmp1, Rstore_addr, Rnew_val);
273 } else {
274 __ z_lgr(Rtmp1, Rstore_addr);
275 __ z_xgr(Rtmp1, Rnew_val);
276 }
277 __ z_srag(Rtmp1, Rtmp1, HeapRegion::LogOfHRGrainBytes);
278 __ z_bre(filtered);
279 }
280
281 // Crosses regions, storing NULL?
282 if (not_null) {
283 #ifdef ASSERT
284 __ z_ltgr(Rnew_val, Rnew_val);
285 __ asm_assert_ne("null oop not allowed (G1 post)", 0x322); // Checked by caller.
286 #endif
287 } else {
288 __ z_ltgr(Rnew_val, Rnew_val);
289 __ z_bre(filtered);
290 }
291
292 Rnew_val = noreg; // end of lifetime
293
294 // Storing region crossing non-NULL, is card already dirty?
295 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
296 assert_different_registers(Rtmp1, Rtmp2, Rtmp3);
297 // Make sure not to use Z_R0 for any of these registers.
298 Register Rcard_addr = (Rtmp1 != Z_R0_scratch) ? Rtmp1 : Rtmp3;
299 Register Rbase = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp3;
300
301 // calculate address of card
302 __ load_const_optimized(Rbase, (address)ct->card_table()->byte_map_base()); // Card table base.
303 __ z_srlg(Rcard_addr, Rstore_addr, CardTable::card_shift); // Index into card table.
304 __ z_algr(Rcard_addr, Rbase); // Explicit calculation needed for cli.
305 Rbase = noreg; // end of lifetime
306
307 // Filter young.
308 assert((unsigned int)G1CardTable::g1_young_card_val() <= 255, "otherwise check this code");
309 __ z_cli(0, Rcard_addr, G1CardTable::g1_young_card_val());
310 __ z_bre(filtered);
311
312 // Check the card value. If dirty, we're done.
313 // This also avoids false sharing of the (already dirty) card.
314 __ z_sync(); // Required to support concurrent cleaning.
315 assert((unsigned int)G1CardTable::dirty_card_val() <= 255, "otherwise check this code");
316 __ z_cli(0, Rcard_addr, G1CardTable::dirty_card_val()); // Reload after membar.
317 __ z_bre(filtered);
318
319 // Storing a region crossing, non-NULL oop, card is clean.
320 // Dirty card and log.
321 __ z_mvi(0, Rcard_addr, G1CardTable::dirty_card_val());
322
323 Register Rcard_addr_x = Rcard_addr;
324 Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
325 Register Rqueue_buf = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1;
326 const int qidx_off = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
327 const int qbuf_off = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
328 if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) {
329 Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0.
330 }
331 __ lgr_if_needed(Rcard_addr_x, Rcard_addr);
332
333 __ load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off));
334 __ z_bre(callRuntime); // Index == 0 then jump to runtime.
335
336 __ z_lg(Rqueue_buf, qbuf_off, Z_thread);
337
338 __ add2reg(Rqueue_index, -wordSize); // Decrement index.
339 __ z_stg(Rqueue_index, qidx_off, Z_thread);
340
341 __ z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card.
342 __ z_bru(filtered);
343
344 __ bind(callRuntime);
345
346 // TODO: do we need a frame? Introduced to be on the safe side.
347 bool needs_frame = true;
348 __ lgr_if_needed(Rcard_addr, Rcard_addr_x); // copy back asap. push_frame will destroy Z_R0_scratch!
349
350 // VM call need frame to access(write) O register.
351 if (needs_frame) {
352 __ save_return_pc();
353 __ push_frame_abi160(0); // Will use Z_R0 as tmp on old CPUs.
354 }
355
356 // Save the live input values.
357 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), Rcard_addr, Z_thread);
358
359 if (needs_frame) {
360 __ pop_frame();
361 __ restore_return_pc();
362 }
363
364 __ bind(filtered);
365
366 BLOCK_COMMENT("} g1_write_barrier_post");
367 }
368
369 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
370 const Address& dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
371 bool on_array = (decorators & IN_HEAP_ARRAY) != 0;
372 bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
373 bool precise = on_array || on_anonymous;
374 // Load and record the previous value.
375 g1_write_barrier_pre(masm, decorators, &dst, tmp3, val, tmp1, tmp2, false);
376
377 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
378
379 // No need for post barrier if storing NULL
380 if (val != noreg) {
381 const Register base = dst.base(),
382 idx = dst.index();
383 const intptr_t disp = dst.disp();
384 if (precise && (disp != 0 || idx != noreg)) {
385 __ add2reg_with_index(base, disp, idx, base);
386 }
387 g1_write_barrier_post(masm, decorators, base, val, tmp1, tmp2, tmp3);
388 }
389 }
390
391 void G1BarrierSetAssembler::resolve_jobject(MacroAssembler* masm, Register value, Register tmp1, Register tmp2) {
392 NearLabel Ldone, Lnot_weak;
393 __ z_ltgr(tmp1, value);
394 __ z_bre(Ldone); // Use NULL result as-is.
395
396 __ z_nill(value, ~JNIHandles::weak_tag_mask);
397 __ z_lg(value, 0, value); // Resolve (untagged) jobject.
398
399 __ z_tmll(tmp1, JNIHandles::weak_tag_mask); // Test for jweak tag.
400 __ z_braz(Lnot_weak);
401 __ verify_oop(value);
402 DecoratorSet decorators = IN_ROOT | ON_PHANTOM_OOP_REF;
403 g1_write_barrier_pre(masm, decorators, (const Address*)NULL, value, noreg, tmp1, tmp2, true);
404 __ bind(Lnot_weak);
405 __ verify_oop(value);
406 __ bind(Ldone);
407 }
408
409 #undef __