1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "gc/g1/g1BarrierSet.hpp"
28 #include "gc/g1/g1BarrierSetAssembler.hpp"
29 #include "gc/g1/g1CardTable.hpp"
30 #include "gc/g1/g1ThreadLocalData.hpp"
31 #include "gc/g1/heapRegion.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "utilities/macros.hpp"
35
36 #define __ masm->
37
38 void G1BarrierSetAssembler::gen_write_ref_array_pre_barrier(MacroAssembler* masm, DecoratorSet decorators,
39 Register addr, Register count) {
40 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
41
42 if (!dest_uninitialized) {
43 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
44 #ifndef _LP64
45 __ push(thread);
46 __ get_thread(thread);
47 #endif
48
49 Label filtered;
50 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
51 // Is marking active?
52 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
53 __ cmpl(in_progress, 0);
54 } else {
55 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
56 __ cmpb(in_progress, 0);
57 }
58
59 NOT_LP64(__ pop(thread);)
60
61 __ jcc(Assembler::equal, filtered);
62
63 __ pusha(); // push registers
64 #ifdef _LP64
65 if (count == c_rarg0) {
66 if (addr == c_rarg1) {
67 // exactly backwards!!
68 __ xchgptr(c_rarg1, c_rarg0);
69 } else {
70 __ movptr(c_rarg1, count);
71 __ movptr(c_rarg0, addr);
72 }
73 } else {
74 __ movptr(c_rarg0, addr);
75 __ movptr(c_rarg1, count);
76 }
77 if (UseCompressedOops) {
78 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_narrow_oop_entry), 2);
79 } else {
80 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry), 2);
81 }
82 #else
83 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_pre_oop_entry),
84 addr, count);
85 #endif
86 __ popa();
87
88 __ bind(filtered);
89 }
90 }
91
92 void G1BarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
93 Register addr, Register count, Register tmp) {
94 __ pusha(); // push registers (overkill)
95 #ifdef _LP64
96 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
97 assert_different_registers(c_rarg1, addr);
98 __ mov(c_rarg1, count);
99 __ mov(c_rarg0, addr);
100 } else {
101 assert_different_registers(c_rarg0, count);
102 __ mov(c_rarg0, addr);
103 __ mov(c_rarg1, count);
104 }
105 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry), 2);
106 #else
107 __ call_VM_leaf(CAST_FROM_FN_PTR(address, G1BarrierSet::write_ref_array_post_entry),
108 addr, count);
109 #endif
110 __ popa();
111 }
112
113 void G1BarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
114 Register dst, Address src, Register tmp1, Register tmp_thread) {
115 bool on_oop = type == T_OBJECT || type == T_ARRAY;
116 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
117 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
118 bool on_reference = on_weak || on_phantom;
119 ModRefBarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
120 if (on_oop && on_reference) {
121 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
122 NOT_LP64(__ get_thread(thread));
123
124 // Generate the G1 pre-barrier code to log the value of
125 // the referent field in an SATB buffer.
126 g1_write_barrier_pre(masm /* masm */,
127 noreg /* obj */,
128 dst /* pre_val */,
129 thread /* thread */,
130 tmp1 /* tmp */,
131 true /* tosca_live */,
132 true /* expand_call */);
133 }
134 }
135
136 void G1BarrierSetAssembler::g1_write_barrier_pre(MacroAssembler* masm,
137 Register obj,
138 Register pre_val,
139 Register thread,
140 Register tmp,
141 bool tosca_live,
142 bool expand_call) {
143 // If expand_call is true then we expand the call_VM_leaf macro
144 // directly to skip generating the check by
145 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
146
147 #ifdef _LP64
148 assert(thread == r15_thread, "must be");
149 #endif // _LP64
150
151 Label done;
152 Label runtime;
153
154 assert(pre_val != noreg, "check this code");
155
156 if (obj != noreg) {
157 assert_different_registers(obj, pre_val, tmp);
158 assert(pre_val != rax, "check this code");
159 }
160
161 Address in_progress(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
162 Address index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
163 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
164
165 // Is marking active?
166 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
167 __ cmpl(in_progress, 0);
168 } else {
169 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
170 __ cmpb(in_progress, 0);
171 }
172 __ jcc(Assembler::equal, done);
173
174 // Do we need to load the previous value?
175 if (obj != noreg) {
176 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
177 }
178
179 // Is the previous value null?
180 __ cmpptr(pre_val, (int32_t) NULL_WORD);
181 __ jcc(Assembler::equal, done);
182
183 // Can we store original value in the thread's buffer?
184 // Is index == 0?
185 // (The index field is typed as size_t.)
186
187 __ movptr(tmp, index); // tmp := *index_adr
188 __ cmpptr(tmp, 0); // tmp == 0?
189 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
190
191 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
192 __ movptr(index, tmp); // *index_adr := tmp
193 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
194
195 // Record the previous value
196 __ movptr(Address(tmp, 0), pre_val);
197 __ jmp(done);
198
199 __ bind(runtime);
200 // save the live input values
201 if(tosca_live) __ push(rax);
202
203 if (obj != noreg && obj != rax)
204 __ push(obj);
205
206 if (pre_val != rax)
207 __ push(pre_val);
208
209 // Calling the runtime using the regular call_VM_leaf mechanism generates
210 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
211 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
212 //
213 // If we care generating the pre-barrier without a frame (e.g. in the
214 // intrinsified Reference.get() routine) then ebp might be pointing to
215 // the caller frame and so this check will most likely fail at runtime.
216 //
217 // Expanding the call directly bypasses the generation of the check.
218 // So when we do not have have a full interpreter frame on the stack
219 // expand_call should be passed true.
220
221 NOT_LP64( __ push(thread); )
222
223 if (expand_call) {
224 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
225 #ifdef _LP64
226 if (c_rarg1 != thread) {
227 __ mov(c_rarg1, thread);
228 }
229 if (c_rarg0 != pre_val) {
230 __ mov(c_rarg0, pre_val);
231 }
232 #else
233 __ push(thread);
234 __ push(pre_val);
235 #endif
236 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
237 } else {
238 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
239 }
240
241 NOT_LP64( __ pop(thread); )
242
243 // save the live input values
244 if (pre_val != rax)
245 __ pop(pre_val);
246
247 if (obj != noreg && obj != rax)
248 __ pop(obj);
249
250 if(tosca_live) __ pop(rax);
251
252 __ bind(done);
253 }
254
255 void G1BarrierSetAssembler::g1_write_barrier_post(MacroAssembler* masm,
256 Register store_addr,
257 Register new_val,
258 Register thread,
259 Register tmp,
260 Register tmp2) {
261 #ifdef _LP64
262 assert(thread == r15_thread, "must be");
263 #endif // _LP64
264
265 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
266 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
267
268 CardTableBarrierSet* ct =
269 barrier_set_cast<CardTableBarrierSet>(BarrierSet::barrier_set());
270 assert(sizeof(*ct->card_table()->byte_map_base()) == sizeof(jbyte), "adjust this code");
271
272 Label done;
273 Label runtime;
274
275 // Does store cross heap regions?
276
277 __ movptr(tmp, store_addr);
278 __ xorptr(tmp, new_val);
279 __ shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
280 __ jcc(Assembler::equal, done);
281
282 // crosses regions, storing NULL?
283
284 __ cmpptr(new_val, (int32_t) NULL_WORD);
285 __ jcc(Assembler::equal, done);
286
287 // storing region crossing non-NULL, is card already dirty?
288
289 const Register card_addr = tmp;
290 const Register cardtable = tmp2;
291
292 __ movptr(card_addr, store_addr);
293 __ shrptr(card_addr, CardTable::card_shift);
294 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
295 // a valid address and therefore is not properly handled by the relocation code.
296 __ movptr(cardtable, (intptr_t)ct->card_table()->byte_map_base());
297 __ addptr(card_addr, cardtable);
298
299 __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
300 __ jcc(Assembler::equal, done);
301
302 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
303 __ cmpb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
304 __ jcc(Assembler::equal, done);
305
306
307 // storing a region crossing, non-NULL oop, card is clean.
308 // dirty card and log.
309
310 __ movb(Address(card_addr, 0), (int)G1CardTable::dirty_card_val());
311
312 __ cmpl(queue_index, 0);
313 __ jcc(Assembler::equal, runtime);
314 __ subl(queue_index, wordSize);
315 __ movptr(tmp2, buffer);
316 #ifdef _LP64
317 __ movslq(rscratch1, queue_index);
318 __ addq(tmp2, rscratch1);
319 __ movq(Address(tmp2, 0), card_addr);
320 #else
321 __ addl(tmp2, queue_index);
322 __ movl(Address(tmp2, 0), card_addr);
323 #endif
324 __ jmp(done);
325
326 __ bind(runtime);
327 // save the live input values
328 __ push(store_addr);
329 __ push(new_val);
330 #ifdef _LP64
331 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
332 #else
333 __ push(thread);
334 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
335 __ pop(thread);
336 #endif
337 __ pop(new_val);
338 __ pop(store_addr);
339
340 __ bind(done);
341 }
342
343 void G1BarrierSetAssembler::oop_store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
344 Address dst, Register val, Register tmp1, Register tmp2) {
345 bool in_heap = (decorators & IN_HEAP) != 0;
346 bool in_concurrent_root = (decorators & IN_CONCURRENT_ROOT) != 0;
347
348 bool needs_pre_barrier = in_heap || in_concurrent_root;
349 bool needs_post_barrier = val != noreg && in_heap;
350
351 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
352 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
353 // flatten object address if needed
354 // We do it regardless of precise because we need the registers
355 if (dst.index() == noreg && dst.disp() == 0) {
356 if (dst.base() != tmp1) {
357 __ movptr(tmp1, dst.base());
358 }
359 } else {
360 __ lea(tmp1, dst);
361 }
362
363 #ifndef _LP64
364 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
365 #endif
366
367 NOT_LP64(__ get_thread(rcx));
368 NOT_LP64(imasm->save_bcp());
369
370 if (needs_pre_barrier) {
371 g1_write_barrier_pre(masm /*masm*/,
372 tmp1 /* obj */,
373 tmp2 /* pre_val */,
374 rthread /* thread */,
375 tmp3 /* tmp */,
376 val != noreg /* tosca_live */,
377 false /* expand_call */);
378 }
379 if (val == noreg) {
380 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
381 } else {
382 Register new_val = val;
383 if (needs_post_barrier) {
384 // G1 barrier needs uncompressed oop for region cross check.
385 if (UseCompressedOops) {
386 new_val = tmp2;
387 __ movptr(new_val, val);
388 }
389 }
390 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
391 if (needs_post_barrier) {
392 g1_write_barrier_post(masm /*masm*/,
393 tmp1 /* store_adr */,
394 new_val /* new_val */,
395 rthread /* thread */,
396 tmp3 /* tmp */,
397 tmp2 /* tmp2 */);
398 }
399 }
400 NOT_LP64(imasm->restore_bcp());
401 }