1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
27 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
28 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
29 #include "gc/shenandoah/shenandoahRuntime.hpp"
30 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "interpreter/interp_masm.hpp"
33 #include "runtime/sharedRuntime.hpp"
34 #include "runtime/thread.hpp"
35 #include "utilities/macros.hpp"
36 #ifdef COMPILER1
37 #include "c1/c1_LIRAssembler.hpp"
38 #include "c1/c1_MacroAssembler.hpp"
39 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
40 #endif
41
42 #define __ masm->
43
44 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
45 address ShenandoahBarrierSetAssembler::_shenandoah_wb_C = NULL;
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
48 Register src, Register dst, Register count) {
49
50 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
51 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
52 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
53 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
54
55 if (type == T_OBJECT || type == T_ARRAY) {
56 #ifdef _LP64
57 if (!checkcast && !obj_int) {
58 // Save count for barrier
59 __ movptr(r11, count);
60 } else if (disjoint && obj_int) {
61 // Save dst in r11 in the disjoint case
62 __ movq(r11, dst);
63 }
64 #else
65 if (disjoint) {
66 __ mov(rdx, dst); // save 'to'
67 }
68 #endif
69
70 if (!dest_uninitialized) {
71 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
72 #ifndef _LP64
73 __ push(thread);
74 __ get_thread(thread);
75 #endif
76
77 Label filtered;
78 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
79 // Is marking active?
80 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
81 __ cmpl(in_progress, 0);
82 } else {
83 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
84 __ cmpb(in_progress, 0);
85 }
86
87 NOT_LP64(__ pop(thread);)
88
89 __ jcc(Assembler::equal, filtered);
90
91 __ pusha(); // push registers
92 #ifdef _LP64
93 if (count == c_rarg0) {
94 if (dst == c_rarg1) {
95 // exactly backwards!!
96 __ xchgptr(c_rarg1, c_rarg0);
97 } else {
98 __ movptr(c_rarg1, count);
99 __ movptr(c_rarg0, dst);
100 }
101 } else {
102 __ movptr(c_rarg0, dst);
103 __ movptr(c_rarg1, count);
104 }
105 if (UseCompressedOops) {
106 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
107 } else {
108 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
109 }
110 #else
111 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
112 dst, count);
113 #endif
114 __ popa();
115 __ bind(filtered);
116 }
117 }
118
119 }
120
121 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
122 Register src, Register dst, Register count) {
123 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
124 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
125 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
126 Register tmp = rax;
127
128 if (type == T_OBJECT || type == T_ARRAY) {
129 #ifdef _LP64
130 if (!checkcast && !obj_int) {
131 // Save count for barrier
132 count = r11;
133 } else if (disjoint && obj_int) {
134 // Use the saved dst in the disjoint case
135 dst = r11;
136 } else if (checkcast) {
137 tmp = rscratch1;
138 }
139 #else
140 if (disjoint) {
141 __ mov(dst, rdx); // restore 'to'
142 }
143 #endif
144
145 __ pusha(); // push registers (overkill)
146 #ifdef _LP64
147 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
148 assert_different_registers(c_rarg1, dst);
149 __ mov(c_rarg1, count);
150 __ mov(c_rarg0, dst);
151 } else {
152 assert_different_registers(c_rarg0, count);
153 __ mov(c_rarg0, dst);
154 __ mov(c_rarg1, count);
155 }
156 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
157 #else
158 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
159 dst, count);
160 #endif
161 __ popa();
162 }
163 }
164
165 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
166 Register obj,
167 Register pre_val,
168 Register thread,
169 Register tmp,
170 bool tosca_live,
171 bool expand_call) {
172
173 if (ShenandoahSATBBarrier) {
174 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
175 }
176 }
177
178 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
179 Register obj,
180 Register pre_val,
181 Register thread,
182 Register tmp,
183 bool tosca_live,
184 bool expand_call) {
185 // If expand_call is true then we expand the call_VM_leaf macro
186 // directly to skip generating the check by
187 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
188
189 #ifdef _LP64
190 assert(thread == r15_thread, "must be");
191 #endif // _LP64
192
193 Label done;
194 Label runtime;
195
196 assert(pre_val != noreg, "check this code");
197
198 if (obj != noreg) {
199 assert_different_registers(obj, pre_val, tmp);
200 assert(pre_val != rax, "check this code");
201 }
202
203 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
204 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
205 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
206
207 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
208 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
209 __ jcc(Assembler::zero, done);
210
211 // Do we need to load the previous value?
212 if (obj != noreg) {
213 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
214 }
215
216 // Is the previous value null?
217 __ cmpptr(pre_val, (int32_t) NULL_WORD);
218 __ jcc(Assembler::equal, done);
219
220 // Can we store original value in the thread's buffer?
221 // Is index == 0?
222 // (The index field is typed as size_t.)
223
224 __ movptr(tmp, index); // tmp := *index_adr
225 __ cmpptr(tmp, 0); // tmp == 0?
226 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
227
228 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
229 __ movptr(index, tmp); // *index_adr := tmp
230 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
231
232 // Record the previous value
233 __ movptr(Address(tmp, 0), pre_val);
234 __ jmp(done);
235
236 __ bind(runtime);
237 // save the live input values
238 if(tosca_live) __ push(rax);
239
240 if (obj != noreg && obj != rax)
241 __ push(obj);
242
243 if (pre_val != rax)
244 __ push(pre_val);
245
246 // Calling the runtime using the regular call_VM_leaf mechanism generates
247 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
248 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
249 //
250 // If we care generating the pre-barrier without a frame (e.g. in the
251 // intrinsified Reference.get() routine) then ebp might be pointing to
252 // the caller frame and so this check will most likely fail at runtime.
253 //
254 // Expanding the call directly bypasses the generation of the check.
255 // So when we do not have have a full interpreter frame on the stack
256 // expand_call should be passed true.
257
258 NOT_LP64( __ push(thread); )
259
260 if (expand_call) {
261 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
262 #ifdef _LP64
263 if (c_rarg1 != thread) {
264 __ mov(c_rarg1, thread);
265 }
266 if (c_rarg0 != pre_val) {
267 __ mov(c_rarg0, pre_val);
268 }
269 #else
270 __ push(thread);
271 __ push(pre_val);
272 #endif
273 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
274 } else {
275 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
276 }
277
278 NOT_LP64( __ pop(thread); )
279
280 // save the live input values
281 if (pre_val != rax)
282 __ pop(pre_val);
283
284 if (obj != noreg && obj != rax)
285 __ pop(obj);
286
287 if(tosca_live) __ pop(rax);
288
289 __ bind(done);
290 }
291
292 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_post(MacroAssembler* masm,
293 Register store_addr,
294 Register new_val,
295 Register thread,
296 Register tmp,
297 Register tmp2) {
298 assert(UseShenandoahGC, "why else should we be here?");
299
300 if (! UseShenandoahMatrix) {
301 // No need for that barrier if not using matrix.
302 return;
303 }
304
305 Label done;
306 __ testptr(new_val, new_val);
307 __ jcc(Assembler::zero, done);
308 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
309 address matrix_addr = matrix->matrix_addr();
310 __ movptr(rscratch1, (intptr_t) ShenandoahHeap::heap()->base());
311 // Compute to-region index
312 __ movptr(tmp, new_val);
313 __ subptr(tmp, rscratch1);
314 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
315 // Compute from-region index
316 __ movptr(tmp2, store_addr);
317 __ subptr(tmp2, rscratch1);
318 __ shrptr(tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
319 // Compute matrix index
320 __ imulptr(tmp, tmp, matrix->stride_jint());
321 __ addptr(tmp, tmp2);
322 // Address is _matrix[to * stride + from]
323 __ movptr(rscratch1, (intptr_t) matrix_addr);
324 // Test if the element is already set.
325 __ cmpb(Address(rscratch1, tmp, Address::times_1), 0);
326 __ jcc(Assembler::notEqual, done);
327 // Store true, if not yet set.
328 __ movb(Address(rscratch1, tmp, Address::times_1), 1);
329 __ bind(done);
330 }
331
332 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
333 if (ShenandoahReadBarrier) {
334 read_barrier_impl(masm, dst);
335 }
336 }
337
338 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
339 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
340 Label is_null;
341 __ testptr(dst, dst);
342 __ jcc(Assembler::zero, is_null);
343 read_barrier_not_null_impl(masm, dst);
344 __ bind(is_null);
345 }
346
347 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
348 if (ShenandoahReadBarrier) {
349 read_barrier_not_null_impl(masm, dst);
350 }
351 }
352
353 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
354 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
355 __ movptr(dst, Address(dst, BrooksPointer::byte_offset()));
356 }
357
358
359 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
360 if (ShenandoahWriteBarrier) {
361 write_barrier_impl(masm, dst);
362 }
363 }
364
365 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
366 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
367 #ifdef _LP64
368 assert(dst != rscratch1, "different regs");
369
370 Label done;
371
372 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
373 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
374
375 // Now check if evacuation is in progress.
376 read_barrier_not_null(masm, dst);
377
378 __ jcc(Assembler::zero, done);
379 __ push(rscratch1);
380 __ push(rscratch2);
381
382 __ movptr(rscratch1, dst);
383 __ shrptr(rscratch1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
384 __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
385 __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
386 __ testb(rscratch2, 0x1);
387
388 __ pop(rscratch2);
389 __ pop(rscratch1);
390
391 __ jcc(Assembler::zero, done);
392
393 __ push(rscratch1);
394
395 // Save possibly live regs.
396 if (dst != rax) {
397 __ push(rax);
398 }
399 if (dst != rbx) {
400 __ push(rbx);
401 }
402 if (dst != rcx) {
403 __ push(rcx);
404 }
405 if (dst != rdx) {
406 __ push(rdx);
407 }
408 if (dst != c_rarg1) {
409 __ push(c_rarg1);
410 }
411
412 __ subptr(rsp, 2 * Interpreter::stackElementSize);
413 __ movdbl(Address(rsp, 0), xmm0);
414
415 // Call into runtime
416 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_IRT), dst);
417 __ mov(rscratch1, rax);
418
419 // Restore possibly live regs.
420 __ movdbl(xmm0, Address(rsp, 0));
421 __ addptr(rsp, 2 * Interpreter::stackElementSize);
422
423 if (dst != c_rarg1) {
424 __ pop(c_rarg1);
425 }
426 if (dst != rdx) {
427 __ pop(rdx);
428 }
429 if (dst != rcx) {
430 __ pop(rcx);
431 }
432 if (dst != rbx) {
433 __ pop(rbx);
434 }
435 if (dst != rax) {
436 __ pop(rax);
437 }
438
439 // Move result into dst reg.
440 __ mov(dst, rscratch1);
441
442 __ pop(rscratch1);
443
444 __ bind(done);
445 #else
446 Unimplemented();
447 #endif
448 }
449
450 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
451 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
452 storeval_barrier_impl(masm, dst, tmp);
453 }
454 }
455
456 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
457 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
458
459 if (dst == noreg) return;
460
461 #ifdef _LP64
462 if (ShenandoahStoreValEnqueueBarrier) {
463 Label is_null;
464 __ testptr(dst, dst);
465 __ jcc(Assembler::zero, is_null);
466 write_barrier_impl(masm, dst);
467 __ bind(is_null);
468
469 // The set of registers to be saved+restored is the same as in the write-barrier above.
470 // Those are the commonly used registers in the interpreter.
471 __ pusha();
472 // __ push_callee_saved_registers();
473 __ subptr(rsp, 2 * Interpreter::stackElementSize);
474 __ movdbl(Address(rsp, 0), xmm0);
475
476 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
477 __ movdbl(xmm0, Address(rsp, 0));
478 __ addptr(rsp, 2 * Interpreter::stackElementSize);
479 //__ pop_callee_saved_registers();
480 __ popa();
481 }
482 if (ShenandoahStoreValReadBarrier) {
483 read_barrier_impl(masm, dst);
484 }
485 #else
486 Unimplemented();
487 #endif
488 }
489
490 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
491 Register dst, Address src, Register tmp1, Register tmp_thread) {
492 bool on_oop = type == T_OBJECT || type == T_ARRAY;
493 bool in_heap = (decorators & IN_HEAP) != 0;
494 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
495 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
496 bool on_reference = on_weak || on_phantom;
497 if (in_heap) {
498 read_barrier_not_null(masm, src.base());
499 }
500 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
501 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
502 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
503 NOT_LP64(__ get_thread(thread));
504
505 // Generate the SATB pre-barrier code to log the value of
506 // the referent field in an SATB buffer.
507 shenandoah_write_barrier_pre(masm /* masm */,
508 noreg /* obj */,
509 dst /* pre_val */,
510 thread /* thread */,
511 tmp1 /* tmp */,
512 true /* tosca_live */,
513 true /* expand_call */);
514 }
515 }
516
517 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
518 Address dst, Register val, Register tmp1, Register tmp2) {
519
520 bool in_heap = (decorators & IN_HEAP) != 0;
521 bool in_concurrent_root = (decorators & IN_CONCURRENT_ROOT) != 0;
522 if (in_heap) {
523 write_barrier(masm, dst.base());
524 }
525 if (type == T_OBJECT || type == T_ARRAY) {
526 bool needs_pre_barrier = in_heap || in_concurrent_root;
527 bool needs_post_barrier = val != noreg && in_heap && UseShenandoahMatrix;
528
529 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
530 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
531 // flatten object address if needed
532 // We do it regardless of precise because we need the registers
533 if (dst.index() == noreg && dst.disp() == 0) {
534 if (dst.base() != tmp1) {
535 __ movptr(tmp1, dst.base());
536 }
537 } else {
538 __ lea(tmp1, dst);
539 }
540
541 #ifndef _LP64
542 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
543 #endif
544
545 NOT_LP64(__ get_thread(rcx));
546 NOT_LP64(imasm->save_bcp());
547
548 if (needs_pre_barrier) {
549 shenandoah_write_barrier_pre(masm /*masm*/,
550 tmp1 /* obj */,
551 tmp2 /* pre_val */,
552 rthread /* thread */,
553 tmp3 /* tmp */,
554 val != noreg /* tosca_live */,
555 false /* expand_call */);
556 }
557 if (val == noreg) {
558 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
559 } else {
560 storeval_barrier(masm, val, tmp3);
561 Register new_val = val;
562 if (needs_post_barrier) {
563 if (UseCompressedOops) {
564 new_val = tmp2;
565 __ movptr(new_val, val);
566 }
567 }
568 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
569
570 if (needs_post_barrier) {
571 shenandoah_write_barrier_post(masm /*masm*/,
572 tmp1 /* store_adr */,
573 new_val /* new_val */,
574 rthread /* thread */,
575 tmp3 /* tmp */,
576 tmp2 /* tmp2 */);
577 }
578 }
579 NOT_LP64(imasm->restore_bcp());
580
581 } else {
582 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
583 }
584 }
585
586 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, DecoratorSet decorators, Register op1, Register op2) {
587 __ cmpptr(op1, op2);
588 if (ShenandoahAcmpBarrier) {
589 Label done;
590 __ jccb(Assembler::equal, done);
591 read_barrier(masm, op1);
592 read_barrier(masm, op2);
593 __ cmpptr(op1, op2);
594 __ bind(done);
595 }
596 }
597
598 void ShenandoahBarrierSetAssembler::obj_equals_addr(MacroAssembler* masm, DecoratorSet decorators, Register src1, Address src2) {
599 __ cmpptr(src1, src2);
600 if (ShenandoahAcmpBarrier) {
601 Label done;
602 __ jccb(Assembler::equal, done);
603 __ movptr(rscratch2, src2);
604 read_barrier(masm, src1);
605 read_barrier(masm, rscratch2);
606 __ cmpptr(src1, rscratch2);
607 __ bind(done);
608 }
609 }
610
611 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
612 bool oop_not_null = (decorators & OOP_NOT_NULL) != 0;
613 if (oop_not_null) {
614 read_barrier_not_null(masm, obj);
615 } else {
616 read_barrier(masm, obj);
617 }
618 }
619
620 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
621 write_barrier(masm, obj);
622 }
623
624 // Special Shenandoah CAS implementation that handles false negatives
625 // due to concurrent evacuation.
626 #ifndef _LP64
627 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, DecoratorSet decorators,
628 Register res, Address addr, Register oldval, Register newval,
629 bool exchange, bool encode, Register tmp1, Register tmp2) {
630 // Shenandoah has no 32-bit version for this.
631 Unimplemented();
632 }
633 #else
634 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, DecoratorSet decorators,
635 Register res, Address addr, Register oldval, Register newval,
636 bool exchange, bool encode, Register tmp1, Register tmp2) {
637
638 if (!ShenandoahCASBarrier) {
639 BarrierSetAssembler::cmpxchg_oop(masm, decorators, res, addr, oldval, newval, exchange, encode, tmp1, tmp2);
640 return;
641 }
642
643 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
644 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
645
646 Label retry, done;
647
648 // Apply storeval barrier to newval.
649 if (encode) {
650 if (newval == c_rarg1 && ShenandoahStoreValEnqueueBarrier) {
651 __ mov(tmp2, newval);
652 storeval_barrier(masm, tmp2, tmp1);
653 } else {
654 storeval_barrier(masm, newval, tmp1);
655 }
656 }
657
658 if (UseCompressedOops) {
659 if (encode) {
660 __ encode_heap_oop(oldval);
661 __ mov(rscratch1, newval);
662 __ encode_heap_oop(rscratch1);
663 newval = rscratch1;
664 }
665 }
666
667 // Remember oldval for retry logic below
668 if (UseCompressedOops) {
669 __ movl(tmp1, oldval);
670 } else {
671 __ movptr(tmp1, oldval);
672 }
673
674 // Step 1. Try to CAS with given arguments. If successful, then we are done,
675 // and can safely return.
676 if (os::is_MP()) __ lock();
677 if (UseCompressedOops) {
678 __ cmpxchgl(newval, addr);
679 } else {
680 __ cmpxchgptr(newval, addr);
681 }
682 __ jcc(Assembler::equal, done, true);
683
684 // Step 2. CAS had failed. This may be a false negative.
685 //
686 // The trouble comes when we compare the to-space pointer with the from-space
687 // pointer to the same object. To resolve this, it will suffice to read both
688 // oldval and the value from memory through the read barriers -- this will give
689 // both to-space pointers. If they mismatch, then it was a legitimate failure.
690 //
691 if (UseCompressedOops) {
692 __ decode_heap_oop(tmp1);
693 }
694 __ resolve_for_read(0, tmp1);
695
696 if (UseCompressedOops) {
697 __ movl(tmp2, oldval);
698 __ decode_heap_oop(tmp2);
699 } else {
700 __ movptr(tmp2, oldval);
701 }
702 __ resolve_for_read(0, tmp2);
703
704 __ cmpptr(tmp1, tmp2);
705 __ jcc(Assembler::notEqual, done, true);
706
707 // Step 3. Try to CAS again with resolved to-space pointers.
708 //
709 // Corner case: it may happen that somebody stored the from-space pointer
710 // to memory while we were preparing for retry. Therefore, we can fail again
711 // on retry, and so need to do this in loop, always re-reading the failure
712 // witness through the read barrier.
713 __ bind(retry);
714 if (os::is_MP()) __ lock();
715 if (UseCompressedOops) {
716 __ cmpxchgl(newval, addr);
717 } else {
718 __ cmpxchgptr(newval, addr);
719 }
720 __ jcc(Assembler::equal, done, true);
721
722 if (UseCompressedOops) {
723 __ movl(tmp2, oldval);
724 __ decode_heap_oop(tmp2);
725 } else {
726 __ movptr(tmp2, oldval);
727 }
728 __ resolve_for_read(0, tmp2);
729
730 __ cmpptr(tmp1, tmp2);
731 __ jcc(Assembler::equal, retry, true);
732
733 // Step 4. If we need a boolean result out of CAS, check the flag again,
734 // and promote the result. Note that we handle the flag from both the CAS
735 // itself and from the retry loop.
736 __ bind(done);
737 if (!exchange) {
738 assert(res != NULL, "need result register");
739 __ setb(Assembler::equal, res);
740 __ movzbl(res, res);
741 }
742 }
743 #endif // LP64
744
745 void ShenandoahBarrierSetAssembler::xchg_oop(MacroAssembler* masm, DecoratorSet decorators,
746 Register obj, Address addr, Register tmp) {
747 storeval_barrier(masm, obj, tmp);
748 BarrierSetAssembler::xchg_oop(masm, decorators, obj, addr, tmp);
749 }
750
751 #ifdef COMPILER1
752
753 #undef __
754 #define __ ce->masm()->
755
756 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
757 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
758 // At this point we know that marking is in progress.
759 // If do_load() is true then we have to emit the
760 // load of the previous value; otherwise it has already
761 // been loaded into _pre_val.
762
763 __ bind(*stub->entry());
764 assert(stub->pre_val()->is_register(), "Precondition.");
765
766 Register pre_val_reg = stub->pre_val()->as_register();
767
768 if (stub->do_load()) {
769 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
770 }
771
772 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
773 __ jcc(Assembler::equal, *stub->continuation());
774 ce->store_parameter(stub->pre_val()->as_register(), 0);
775 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
776 __ jmp(*stub->continuation());
777
778 }
779
780 #undef __
781
782 #define __ sasm->
783
784 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
785 __ prologue("shenandoah_pre_barrier", false);
786 // arg0 : previous value of memory
787
788 __ push(rax);
789 __ push(rdx);
790
791 const Register pre_val = rax;
792 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
793 const Register tmp = rdx;
794
795 NOT_LP64(__ get_thread(thread);)
796
797 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
798 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
799
800 Label done;
801 Label runtime;
802
803 // Is SATB still active?
804 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
805 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
806 __ jcc(Assembler::zero, done);
807
808 // Can we store original value in the thread's buffer?
809
810 __ movptr(tmp, queue_index);
811 __ testptr(tmp, tmp);
812 __ jcc(Assembler::zero, runtime);
813 __ subptr(tmp, wordSize);
814 __ movptr(queue_index, tmp);
815 __ addptr(tmp, buffer);
816
817 // prev_val (rax)
818 __ load_parameter(0, pre_val);
819 __ movptr(Address(tmp, 0), pre_val);
820 __ jmp(done);
821
822 __ bind(runtime);
823
824 __ save_live_registers_no_oop_map(true);
825
826 // load the pre-value
827 __ load_parameter(0, rcx);
828 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
829
830 __ restore_live_registers(true);
831
832 __ bind(done);
833
834 __ pop(rdx);
835 __ pop(rax);
836
837 __ epilogue();
838 }
839
840 #undef __
841
842 #endif // COMPILER1
843
844 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
845 return _shenandoah_wb;
846 }
847
848 address ShenandoahBarrierSetAssembler::shenandoah_wb_C() {
849 return _shenandoah_wb_C;
850 }
851
852 #define __ cgen->assembler()->
853
854 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) {
855 __ align(CodeEntryAlignment);
856 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
857 address start = __ pc();
858
859 Label not_done;
860
861 // We use RDI, which also serves as argument register for slow call.
862 // RAX always holds the src object ptr, except after the slow call and
863 // the cmpxchg, then it holds the result.
864 // R8 and RCX are used as temporary registers.
865 if (!c_abi) {
866 __ push(rdi);
867 __ push(r8);
868 }
869
870 // Check for object beeing in the collection set.
871 // TODO: Can we use only 1 register here?
872 // The source object arrives here in rax.
873 // live: rax
874 // live: rdi
875 if (!c_abi) {
876 __ mov(rdi, rax);
877 } else {
878 if (rax != c_rarg0) {
879 __ mov(rax, c_rarg0);
880 }
881 }
882 if (do_cset_test) {
883 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
884 // live: r8
885 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
886 __ movbool(r8, Address(r8, rdi, Address::times_1));
887 // unlive: rdi
888 __ testbool(r8);
889 // unlive: r8
890 __ jccb(Assembler::notZero, not_done);
891
892 if (!c_abi) {
893 __ pop(r8);
894 __ pop(rdi);
895 }
896 __ ret(0);
897
898 __ bind(not_done);
899 }
900
901 if (!c_abi) {
902 __ push(rcx);
903 }
904
905 if (!c_abi) {
906 __ push(rdx);
907 __ push(rdi);
908 __ push(rsi);
909 __ push(r8);
910 __ push(r9);
911 __ push(r10);
912 __ push(r11);
913 __ push(r12);
914 __ push(r13);
915 __ push(r14);
916 __ push(r15);
917 }
918 __ save_vector_registers();
919 __ movptr(rdi, rax);
920 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
921 __ restore_vector_registers();
922 if (!c_abi) {
923 __ pop(r15);
924 __ pop(r14);
925 __ pop(r13);
926 __ pop(r12);
927 __ pop(r11);
928 __ pop(r10);
929 __ pop(r9);
930 __ pop(r8);
931 __ pop(rsi);
932 __ pop(rdi);
933 __ pop(rdx);
934
935 __ pop(rcx);
936 __ pop(r8);
937 __ pop(rdi);
938 }
939 __ ret(0);
940
941 return start;
942 }
943
944 #undef __
945
946 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
947 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
948 int stub_code_size = 1536;
949 ResourceMark rm;
950 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
951 CodeBuffer buf(bb);
952 StubCodeGenerator cgen(&buf);
953 _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true);
954 _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, !ShenandoahWriteBarrierCsetTestInIR);
955 }
956 }