1 /*
2 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "c1/c1_LIRAssembler.hpp"
27 #include "c1/c1_MacroAssembler.hpp"
28 #include "gc/g1/c1_G1BSCodeGen.hpp"
29 #include "gc/g1/g1BSCodeGen.hpp"
30 #include "gc/g1/g1BarrierSet.hpp"
31 #include "gc/g1/g1CardTable.hpp"
32
33 #define __ masm->
34
35 void G1BSCodeGen::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
36 Register addr, Register count, RegSet saved_regs) {
37 bool dest_uninitialized = (decorators & DEST_NOT_INITIALIZED) != 0;
38 if (!dest_uninitialized) {
39 __ push(saved_regs, sp);
40 if (count == c_rarg0) {
41 if (addr == c_rarg1) {
42 // exactly backwards!!
43 __ mov(rscratch1, c_rarg0);
44 __ mov(c_rarg0, c_rarg1);
45 __ mov(c_rarg1, rscratch1);
46 } else {
47 __ mov(c_rarg1, count);
48 __ mov(c_rarg0, addr);
49 }
50 } else {
51 __ mov(c_rarg0, addr);
52 __ mov(c_rarg1, count);
53 }
54 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ModRefBarrierSet::static_write_ref_array_pre), 2);
55 __ pop(saved_regs, sp);
56 }
57 }
58
59 void G1BSCodeGen::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
60 Register start, Register end, Register scratch, RegSet saved_regs) {
61 __ push(saved_regs, sp);
62 // must compute element count unless barrier set interface is changed (other platforms supply count)
63 assert_different_registers(start, end, scratch);
64 __ lea(scratch, Address(end, BytesPerHeapOop));
65 __ sub(scratch, scratch, start); // subtract start to get #bytes
66 __ lsr(scratch, scratch, LogBytesPerHeapOop); // convert to element count
67 __ mov(c_rarg0, start);
68 __ mov(c_rarg1, scratch);
69 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ModRefBarrierSet::static_write_ref_array_post), 2);
70 __ pop(saved_regs, sp);
71 }
72
73 void G1BSCodeGen::g1_write_barrier_pre(MacroAssembler* masm,
74 Register obj,
75 Register pre_val,
76 Register thread,
77 Register tmp,
78 bool tosca_live,
79 bool expand_call) {
80 // If expand_call is true then we expand the call_VM_leaf macro
81 // directly to skip generating the check by
82 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
83
84 assert(thread == rthread, "must be");
85
86 Label done;
87 Label runtime;
88
89 assert_different_registers(obj, pre_val, tmp, rscratch1);
90 assert(pre_val != noreg && tmp != noreg, "expecting a register");
91
92 Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
93 SATBMarkQueue::byte_offset_of_active()));
94 Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
95 SATBMarkQueue::byte_offset_of_index()));
96 Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
97 SATBMarkQueue::byte_offset_of_buf()));
98
99
100 // Is marking active?
101 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
102 __ ldrw(tmp, in_progress);
103 } else {
104 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
105 __ ldrb(tmp, in_progress);
106 }
107 __ cbzw(tmp, done);
108
109 // Do we need to load the previous value?
110 if (obj != noreg) {
111 __ load_heap_oop(pre_val, Address(obj, 0));
112 }
113
114 // Is the previous value null?
115 __ cbz(pre_val, done);
116
117 // Can we store original value in the thread's buffer?
118 // Is index == 0?
119 // (The index field is typed as size_t.)
120
121 __ ldr(tmp, index); // tmp := *index_adr
122 __ cbz(tmp, runtime); // tmp == 0?
123 // If yes, goto runtime
124
125 __ sub(tmp, tmp, wordSize); // tmp := tmp - wordSize
126 __ str(tmp, index); // *index_adr := tmp
127 __ ldr(rscratch1, buffer);
128 __ add(tmp, tmp, rscratch1); // tmp := tmp + *buffer_adr
129
130 // Record the previous value
131 __ str(pre_val, Address(tmp, 0));
132 __ b(done);
133
134 __ bind(runtime);
135 // save the live input values
136 RegSet saved = RegSet::of(pre_val);
137 if (tosca_live) saved += RegSet::of(r0);
138 if (obj != noreg) saved += RegSet::of(obj);
139
140 __ push(saved, sp);
141
142 // Calling the runtime using the regular call_VM_leaf mechanism generates
143 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
144 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
145 //
146 // If we care generating the pre-barrier without a frame (e.g. in the
147 // intrinsified Reference.get() routine) then ebp might be pointing to
148 // the caller frame and so this check will most likely fail at runtime.
149 //
150 // Expanding the call directly bypasses the generation of the check.
151 // So when we do not have have a full interpreter frame on the stack
152 // expand_call should be passed true.
153
154 if (expand_call) {
155 assert(pre_val != c_rarg1, "smashed arg");
156 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread);
157 } else {
158 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread);
159 }
160
161 __ pop(saved, sp);
162
163 __ bind(done);
164
165 }
166
167 void G1BSCodeGen::g1_write_barrier_post(MacroAssembler* masm,
168 Register store_addr,
169 Register new_val,
170 Register thread,
171 Register tmp,
172 Register tmp2) {
173 assert(thread == rthread, "must be");
174 assert_different_registers(store_addr, new_val, thread, tmp, tmp2,
175 rscratch1);
176 assert(store_addr != noreg && new_val != noreg && tmp != noreg
177 && tmp2 != noreg, "expecting a register");
178
179 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
180 DirtyCardQueue::byte_offset_of_index()));
181 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
182 DirtyCardQueue::byte_offset_of_buf()));
183
184 BarrierSet* bs = Universe::heap()->barrier_set();
185 CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
186 CardTable* ct = ctbs->card_table();
187 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
188
189 Label done;
190 Label runtime;
191
192 // Does store cross heap regions?
193
194 __ eor(tmp, store_addr, new_val);
195 __ lsr(tmp, tmp, HeapRegion::LogOfHRGrainBytes);
196 __ cbz(tmp, done);
197
198 // crosses regions, storing NULL?
199
200 __ cbz(new_val, done);
201
202 // storing region crossing non-NULL, is card already dirty?
203
204 ExternalAddress cardtable((address) ct->byte_map_base());
205 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
206 const Register card_addr = tmp;
207
208 __ lsr(card_addr, store_addr, CardTable::card_shift);
209
210 // get the address of the card
211 __ load_byte_map_base(tmp2);
212 __ add(card_addr, card_addr, tmp2);
213 __ ldrb(tmp2, Address(card_addr));
214 __ cmpw(tmp2, (int)G1CardTable::g1_young_card_val());
215 __ br(Assembler::EQ, done);
216
217 assert((int)CardTable::dirty_card_val() == 0, "must be 0");
218
219 __ membar(Assembler::StoreLoad);
220
221 __ ldrb(tmp2, Address(card_addr));
222 __ cbzw(tmp2, done);
223
224 // storing a region crossing, non-NULL oop, card is clean.
225 // dirty card and log.
226
227 __ strb(zr, Address(card_addr));
228
229 __ ldr(rscratch1, queue_index);
230 __ cbz(rscratch1, runtime);
231 __ sub(rscratch1, rscratch1, wordSize);
232 __ str(rscratch1, queue_index);
233
234 __ ldr(tmp2, buffer);
235 __ str(card_addr, Address(tmp2, rscratch1));
236 __ b(done);
237
238 __ bind(runtime);
239 // save the live input values
240 RegSet saved = RegSet::of(store_addr, new_val);
241 __ push(saved, sp);
242 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_post), card_addr, thread);
243 __ pop(saved, sp);
244
245 __ bind(done);
246 }
247
248 void G1BSCodeGen::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
249 Register dst, Address src, Register tmp1, Register tmp_thread) {
250 bool on_oop = type == T_OBJECT || type == T_ARRAY;
251 bool on_weak = (decorators & ACCESS_ON_WEAK_OOP_REF) != 0;
252 bool on_phantom = (decorators & ACCESS_ON_PHANTOM_OOP_REF) != 0;
253 bool on_reference = on_weak || on_phantom;
254 ModRefBSCodeGen::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
255 if (on_oop && on_reference) {
256 // Generate the G1 pre-barrier code to log the value of
257 // the referent field in an SATB buffer.
258 g1_write_barrier_pre(masm /* masm */,
259 noreg /* obj */,
260 dst /* pre_val */,
261 rthread /* thread */,
262 tmp1 /* tmp */,
263 true /* tosca_live */,
264 true /* expand_call */);
265 }
266 }
267
268 void G1BSCodeGen::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
269 Address dst, Register val, Register tmp1, Register tmp2) {
270 // flatten object address if needed
271 if (dst.index() == noreg && dst.offset() == 0) {
272 if (dst.base() != r3) {
273 __ mov(r3, dst.base());
274 }
275 } else {
276 __ lea(r3, dst);
277 }
278
279 g1_write_barrier_pre(masm,
280 r3 /* obj */,
281 tmp2 /* pre_val */,
282 rthread /* thread */,
283 tmp1 /* tmp */,
284 val != noreg /* tosca_live */,
285 false /* expand_call */);
286
287 if (val == noreg) {
288 __ store_heap_oop_null(Address(r3, 0));
289 } else {
290 // G1 barrier needs uncompressed oop for region cross check.
291 Register new_val = val;
292 if (UseCompressedOops) {
293 new_val = rscratch2;
294 __ mov(new_val, val);
295 }
296 __ store_heap_oop(Address(r3, 0), val);
297 g1_write_barrier_post(masm,
298 r3 /* store_adr */,
299 new_val /* new_val */,
300 rthread /* thread */,
301 tmp1 /* tmp */,
302 tmp2 /* tmp2 */);
303 }
304
305 }
306
307 #undef __
308
309 #define __ ce->masm()->
310
311 void G1BSCodeGen::gen_g1_pre_barrier_stub(LIR_Assembler* ce, G1PreBarrierStub* stub) {
312 C1G1BSCodeGen* code_gen = (C1G1BSCodeGen*)Universe::heap()->barrier_set()->c1_code_gen();
313 // At this point we know that marking is in progress.
314 // If do_load() is true then we have to emit the
315 // load of the previous value; otherwise it has already
316 // been loaded into _pre_val.
317
318 __ bind(stub->_entry);
319 assert(stub->pre_val()->is_register(), "Precondition.");
320
321 Register pre_val_reg = stub->pre_val()->as_register();
322
323 if (stub->do_load()) {
324 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
325 }
326 __ cbz(pre_val_reg, stub->_continuation);
327 ce->store_parameter(stub->pre_val()->as_register(), 0);
328 __ far_call(RuntimeAddress(code_gen->pre_barrier_c1_runtime_code_blob()->code_begin()));
329 __ b(stub->_continuation);
330 }
331
332 void G1BSCodeGen::gen_g1_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub) {
333 C1G1BSCodeGen* code_gen = (C1G1BSCodeGen*)Universe::heap()->barrier_set()->c1_code_gen();
334 __ bind(stub->_entry);
335 assert(stub->addr()->is_register(), "Precondition.");
336 assert(stub->new_val()->is_register(), "Precondition.");
337 Register new_val_reg = stub->new_val()->as_register();
338 __ cbz(new_val_reg, stub->_continuation);
339 ce->store_parameter(stub->addr()->as_pointer_register(), 0);
340 __ far_call(RuntimeAddress(code_gen->post_barrier_c1_runtime_code_blob()->code_begin()));
341 __ b(stub->_continuation);
342 }
343
344 #undef __
345
346 #define __ sasm->
347
348 void G1BSCodeGen::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
349 __ prologue("g1_pre_barrier", false);
350
351 // arg0 : previous value of memory
352
353 BarrierSet* bs = Universe::heap()->barrier_set();
354
355 const Register pre_val = r0;
356 const Register thread = rthread;
357 const Register tmp = rscratch1;
358
359 Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
360 SATBMarkQueue::byte_offset_of_active()));
361
362 Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
363 SATBMarkQueue::byte_offset_of_index()));
364 Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
365 SATBMarkQueue::byte_offset_of_buf()));
366
367 Label done;
368 Label runtime;
369
370 // Is marking still active?
371 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
372 __ ldrw(tmp, in_progress);
373 } else {
374 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
375 __ ldrb(tmp, in_progress);
376 }
377 __ cbzw(tmp, done);
378
379 // Can we store original value in the thread's buffer?
380 __ ldr(tmp, queue_index);
381 __ cbz(tmp, runtime);
382
383 __ sub(tmp, tmp, wordSize);
384 __ str(tmp, queue_index);
385 __ ldr(rscratch2, buffer);
386 __ add(tmp, tmp, rscratch2);
387 __ load_parameter(0, rscratch2);
388 __ str(rscratch2, Address(tmp, 0));
389 __ b(done);
390
391 __ bind(runtime);
392 __ push_call_clobbered_registers();
393 __ load_parameter(0, pre_val);
394 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_pre), pre_val, thread);
395 __ pop_call_clobbered_registers();
396 __ bind(done);
397
398 __ epilogue();
399 }
400
401 void G1BSCodeGen::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm) {
402 __ prologue("g1_post_barrier", false);
403
404 // arg0: store_address
405 Address store_addr(rfp, 2*BytesPerWord);
406
407 BarrierSet* bs = Universe::heap()->barrier_set();
408 CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
409 CardTable* ct = ctbs->card_table();
410 assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
411
412 Label done;
413 Label runtime;
414
415 // At this point we know new_value is non-NULL and the new_value crosses regions.
416 // Must check to see if card is already dirty
417
418 const Register thread = rthread;
419
420 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
421 DirtyCardQueue::byte_offset_of_index()));
422 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
423 DirtyCardQueue::byte_offset_of_buf()));
424
425 const Register card_offset = rscratch2;
426 // LR is free here, so we can use it to hold the byte_map_base.
427 const Register byte_map_base = lr;
428
429 assert_different_registers(card_offset, byte_map_base, rscratch1);
430
431 __ load_parameter(0, card_offset);
432 __ lsr(card_offset, card_offset, CardTable::card_shift);
433 __ load_byte_map_base(byte_map_base);
434 __ ldrb(rscratch1, Address(byte_map_base, card_offset));
435 __ cmpw(rscratch1, (int)G1CardTable::g1_young_card_val());
436 __ br(Assembler::EQ, done);
437
438 assert((int)CardTable::dirty_card_val() == 0, "must be 0");
439
440 __ membar(Assembler::StoreLoad);
441 __ ldrb(rscratch1, Address(byte_map_base, card_offset));
442 __ cbzw(rscratch1, done);
443
444 // storing region crossing non-NULL, card is clean.
445 // dirty card and log.
446 __ strb(zr, Address(byte_map_base, card_offset));
447
448 // Convert card offset into an address in card_addr
449 Register card_addr = card_offset;
450 __ add(card_addr, byte_map_base, card_addr);
451
452 __ ldr(rscratch1, queue_index);
453 __ cbz(rscratch1, runtime);
454 __ sub(rscratch1, rscratch1, wordSize);
455 __ str(rscratch1, queue_index);
456
457 // Reuse LR to hold buffer_addr
458 const Register buffer_addr = lr;
459
460 __ ldr(buffer_addr, buffer);
461 __ str(card_addr, Address(buffer_addr, rscratch1));
462 __ b(done);
463
464 __ bind(runtime);
465 __ push_call_clobbered_registers();
466 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::g1_wb_post), card_addr, thread);
467 __ pop_call_clobbered_registers();
468 __ bind(done);
469 __ epilogue();
470 }
471
472 #undef __