1 /*
2 * Copyright (c) 2018, Red Hat, Inc. and/or its affiliates.
3 *
4 * This code is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License version 2 only, as
6 * published by the Free Software Foundation.
7 *
8 * This code is distributed in the hope that it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 * version 2 for more details (a copy is included in the LICENSE file that
12 * accompanied this code).
13 *
14 * You should have received a copy of the GNU General Public License version
15 * 2 along with this work; if not, write to the Free Software Foundation,
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
27 #include "gc/shenandoah/shenandoahConnectionMatrix.hpp"
28 #include "gc/shenandoah/shenandoahHeap.hpp"
29 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
30 #include "gc/shenandoah/shenandoahHeuristics.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "interpreter/interpreter.hpp"
34 #include "interpreter/interp_masm.hpp"
35 #include "runtime/sharedRuntime.hpp"
36 #include "runtime/thread.hpp"
37 #include "utilities/macros.hpp"
38 #ifdef COMPILER1
39 #include "c1/c1_LIRAssembler.hpp"
40 #include "c1/c1_MacroAssembler.hpp"
41 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
42 #endif
43
44 #define __ masm->
45
46 address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
47 address ShenandoahBarrierSetAssembler::_shenandoah_wb_C = NULL;
48
49 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
50 Register src, Register dst, Register count) {
51
52 if (ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
53 // Don't need arraycopy-pre-barrier for traversal. No need to pointlessly
54 // call out to runtime.
55 return;
56 }
57 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
58 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
59 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
60 bool dest_uninitialized = (decorators & AS_DEST_NOT_INITIALIZED) != 0;
61
62 if (type == T_OBJECT || type == T_ARRAY) {
63 #ifdef _LP64
64 if (!checkcast && !obj_int) {
65 // Save count for barrier
66 __ movptr(r11, count);
67 } else if (disjoint && obj_int) {
68 // Save dst in r11 in the disjoint case
69 __ movq(r11, dst);
70 }
71 #else
72 if (disjoint) {
73 __ mov(rdx, dst); // save 'to'
74 }
75 #endif
76
77 if (!dest_uninitialized) {
78 Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
79 #ifndef _LP64
80 __ push(thread);
81 __ get_thread(thread);
82 #endif
83
84 Label filtered;
85 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
86 // Is marking active?
87 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
88 __ cmpl(in_progress, 0);
89 } else {
90 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
91 __ cmpb(in_progress, 0);
92 }
93
94 NOT_LP64(__ pop(thread);)
95
96 __ jcc(Assembler::equal, filtered);
97
98 __ pusha(); // push registers
99 #ifdef _LP64
100 if (count == c_rarg0) {
101 if (dst == c_rarg1) {
102 // exactly backwards!!
103 __ xchgptr(c_rarg1, c_rarg0);
104 } else {
105 __ movptr(c_rarg1, count);
106 __ movptr(c_rarg0, dst);
107 }
108 } else {
109 __ movptr(c_rarg0, dst);
110 __ movptr(c_rarg1, count);
111 }
112 if (UseCompressedOops) {
113 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
114 } else {
115 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
116 }
117 #else
118 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
119 dst, count);
120 #endif
121 __ popa();
122 __ bind(filtered);
123 }
124 }
125
126 }
127
128 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
129 Register src, Register dst, Register count) {
130 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
131 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
132 bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
133 Register tmp = rax;
134
135 if (type == T_OBJECT || type == T_ARRAY) {
136 #ifdef _LP64
137 if (!checkcast && !obj_int) {
138 // Save count for barrier
139 count = r11;
140 } else if (disjoint && obj_int) {
141 // Use the saved dst in the disjoint case
142 dst = r11;
143 } else if (checkcast) {
144 tmp = rscratch1;
145 }
146 #else
147 if (disjoint) {
148 __ mov(dst, rdx); // restore 'to'
149 }
150 #endif
151
152 __ pusha(); // push registers (overkill)
153 #ifdef _LP64
154 if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
155 assert_different_registers(c_rarg1, dst);
156 __ mov(c_rarg1, count);
157 __ mov(c_rarg0, dst);
158 } else {
159 assert_different_registers(c_rarg0, count);
160 __ mov(c_rarg0, dst);
161 __ mov(c_rarg1, count);
162 }
163 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
164 #else
165 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
166 dst, count);
167 #endif
168 __ popa();
169 }
170 }
171
172 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
173 Register obj,
174 Register pre_val,
175 Register thread,
176 Register tmp,
177 bool tosca_live,
178 bool expand_call) {
179
180 if (ShenandoahSATBBarrier) {
181 satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
182 }
183 }
184
185 void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
186 Register obj,
187 Register pre_val,
188 Register thread,
189 Register tmp,
190 bool tosca_live,
191 bool expand_call) {
192 // If expand_call is true then we expand the call_VM_leaf macro
193 // directly to skip generating the check by
194 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
195
196 #ifdef _LP64
197 assert(thread == r15_thread, "must be");
198 #endif // _LP64
199
200 Label done;
201 Label runtime;
202
203 assert(pre_val != noreg, "check this code");
204
205 if (obj != noreg) {
206 assert_different_registers(obj, pre_val, tmp);
207 assert(pre_val != rax, "check this code");
208 }
209
210 Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
211 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
212 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
213
214 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
215 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
216 __ jcc(Assembler::zero, done);
217
218 // Do we need to load the previous value?
219 if (obj != noreg) {
220 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
221 }
222
223 // Is the previous value null?
224 __ cmpptr(pre_val, (int32_t) NULL_WORD);
225 __ jcc(Assembler::equal, done);
226
227 // Can we store original value in the thread's buffer?
228 // Is index == 0?
229 // (The index field is typed as size_t.)
230
231 __ movptr(tmp, index); // tmp := *index_adr
232 __ cmpptr(tmp, 0); // tmp == 0?
233 __ jcc(Assembler::equal, runtime); // If yes, goto runtime
234
235 __ subptr(tmp, wordSize); // tmp := tmp - wordSize
236 __ movptr(index, tmp); // *index_adr := tmp
237 __ addptr(tmp, buffer); // tmp := tmp + *buffer_adr
238
239 // Record the previous value
240 __ movptr(Address(tmp, 0), pre_val);
241 __ jmp(done);
242
243 __ bind(runtime);
244 // save the live input values
245 if(tosca_live) __ push(rax);
246
247 if (obj != noreg && obj != rax)
248 __ push(obj);
249
250 if (pre_val != rax)
251 __ push(pre_val);
252
253 // Calling the runtime using the regular call_VM_leaf mechanism generates
254 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
255 // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
256 //
257 // If we care generating the pre-barrier without a frame (e.g. in the
258 // intrinsified Reference.get() routine) then ebp might be pointing to
259 // the caller frame and so this check will most likely fail at runtime.
260 //
261 // Expanding the call directly bypasses the generation of the check.
262 // So when we do not have have a full interpreter frame on the stack
263 // expand_call should be passed true.
264
265 NOT_LP64( __ push(thread); )
266
267 if (expand_call) {
268 LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
269 #ifdef _LP64
270 if (c_rarg1 != thread) {
271 __ mov(c_rarg1, thread);
272 }
273 if (c_rarg0 != pre_val) {
274 __ mov(c_rarg0, pre_val);
275 }
276 #else
277 __ push(thread);
278 __ push(pre_val);
279 #endif
280 __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
281 } else {
282 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
283 }
284
285 NOT_LP64( __ pop(thread); )
286
287 // save the live input values
288 if (pre_val != rax)
289 __ pop(pre_val);
290
291 if (obj != noreg && obj != rax)
292 __ pop(obj);
293
294 if(tosca_live) __ pop(rax);
295
296 __ bind(done);
297 }
298
299 void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_post(MacroAssembler* masm,
300 Register store_addr,
301 Register new_val,
302 Register thread,
303 Register tmp,
304 Register tmp2) {
305 assert(UseShenandoahGC, "why else should we be here?");
306
307 if (! UseShenandoahMatrix) {
308 // No need for that barrier if not using matrix.
309 return;
310 }
311
312 Label done;
313 __ testptr(new_val, new_val);
314 __ jcc(Assembler::zero, done);
315 ShenandoahConnectionMatrix* matrix = ShenandoahHeap::heap()->connection_matrix();
316 address matrix_addr = matrix->matrix_addr();
317 __ movptr(rscratch1, (intptr_t) ShenandoahHeap::heap()->base());
318 // Compute to-region index
319 __ movptr(tmp, new_val);
320 __ subptr(tmp, rscratch1);
321 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
322 // Compute from-region index
323 __ movptr(tmp2, store_addr);
324 __ subptr(tmp2, rscratch1);
325 __ shrptr(tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
326 // Compute matrix index
327 __ imulptr(tmp, tmp, matrix->stride_jint());
328 __ addptr(tmp, tmp2);
329 // Address is _matrix[to * stride + from]
330 __ movptr(rscratch1, (intptr_t) matrix_addr);
331 // Test if the element is already set.
332 __ cmpb(Address(rscratch1, tmp, Address::times_1), 0);
333 __ jcc(Assembler::notEqual, done);
334 // Store true, if not yet set.
335 __ movb(Address(rscratch1, tmp, Address::times_1), 1);
336 __ bind(done);
337 }
338
339 void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
340 if (ShenandoahReadBarrier) {
341 read_barrier_impl(masm, dst);
342 }
343 }
344
345 void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
346 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
347 Label is_null;
348 __ testptr(dst, dst);
349 __ jcc(Assembler::zero, is_null);
350 read_barrier_not_null_impl(masm, dst);
351 __ bind(is_null);
352 }
353
354 void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
355 if (ShenandoahReadBarrier) {
356 read_barrier_not_null_impl(masm, dst);
357 }
358 }
359
360 void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
361 assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier), "should be enabled");
362 __ movptr(dst, Address(dst, BrooksPointer::byte_offset()));
363 }
364
365
366 void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
367 if (ShenandoahWriteBarrier) {
368 write_barrier_impl(masm, dst);
369 }
370 }
371
372 void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
373 assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
374 #ifdef _LP64
375 assert(dst != rscratch1, "different regs");
376
377 Label done;
378
379 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
380 __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
381
382 // Now check if evacuation is in progress.
383 read_barrier_not_null(masm, dst);
384
385 __ jcc(Assembler::zero, done);
386 __ push(rscratch1);
387 __ push(rscratch2);
388
389 __ movptr(rscratch1, dst);
390 __ shrptr(rscratch1, ShenandoahHeapRegion::region_size_bytes_shift_jint());
391 __ movptr(rscratch2, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
392 __ movbool(rscratch2, Address(rscratch2, rscratch1, Address::times_1));
393 __ testb(rscratch2, 0x1);
394
395 __ pop(rscratch2);
396 __ pop(rscratch1);
397
398 __ jcc(Assembler::zero, done);
399
400 __ push(rscratch1);
401
402 // Save possibly live regs.
403 if (dst != rax) {
404 __ push(rax);
405 }
406 if (dst != rbx) {
407 __ push(rbx);
408 }
409 if (dst != rcx) {
410 __ push(rcx);
411 }
412 if (dst != rdx) {
413 __ push(rdx);
414 }
415 if (dst != c_rarg1) {
416 __ push(c_rarg1);
417 }
418
419 __ subptr(rsp, 2 * Interpreter::stackElementSize);
420 __ movdbl(Address(rsp, 0), xmm0);
421
422 // Call into runtime
423 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_IRT), dst);
424 __ mov(rscratch1, rax);
425
426 // Restore possibly live regs.
427 __ movdbl(xmm0, Address(rsp, 0));
428 __ addptr(rsp, 2 * Interpreter::stackElementSize);
429
430 if (dst != c_rarg1) {
431 __ pop(c_rarg1);
432 }
433 if (dst != rdx) {
434 __ pop(rdx);
435 }
436 if (dst != rcx) {
437 __ pop(rcx);
438 }
439 if (dst != rbx) {
440 __ pop(rbx);
441 }
442 if (dst != rax) {
443 __ pop(rax);
444 }
445
446 // Move result into dst reg.
447 __ mov(dst, rscratch1);
448
449 __ pop(rscratch1);
450
451 __ bind(done);
452 #else
453 Unimplemented();
454 #endif
455 }
456
457 void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
458 if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
459 storeval_barrier_impl(masm, dst, tmp);
460 }
461 }
462
463 void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
464 assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
465
466 if (dst == noreg) return;
467
468 #ifdef _LP64
469 if (ShenandoahStoreValEnqueueBarrier) {
470 Label is_null;
471 __ testptr(dst, dst);
472 __ jcc(Assembler::zero, is_null);
473 write_barrier_impl(masm, dst);
474 __ bind(is_null);
475
476 // The set of registers to be saved+restored is the same as in the write-barrier above.
477 // Those are the commonly used registers in the interpreter.
478 __ pusha();
479 // __ push_callee_saved_registers();
480 __ subptr(rsp, 2 * Interpreter::stackElementSize);
481 __ movdbl(Address(rsp, 0), xmm0);
482
483 satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
484 __ movdbl(xmm0, Address(rsp, 0));
485 __ addptr(rsp, 2 * Interpreter::stackElementSize);
486 //__ pop_callee_saved_registers();
487 __ popa();
488 }
489 if (ShenandoahStoreValReadBarrier) {
490 read_barrier_impl(masm, dst);
491 }
492 #else
493 Unimplemented();
494 #endif
495 }
496
497 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
498 Register dst, Address src, Register tmp1, Register tmp_thread) {
499 bool on_oop = type == T_OBJECT || type == T_ARRAY;
500 bool in_heap = (decorators & IN_HEAP) != 0;
501 bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
502 bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
503 bool on_reference = on_weak || on_phantom;
504 if (in_heap) {
505 read_barrier_not_null(masm, src.base());
506 }
507 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
508 if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
509 const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
510 NOT_LP64(__ get_thread(thread));
511
512 // Generate the SATB pre-barrier code to log the value of
513 // the referent field in an SATB buffer.
514 shenandoah_write_barrier_pre(masm /* masm */,
515 noreg /* obj */,
516 dst /* pre_val */,
517 thread /* thread */,
518 tmp1 /* tmp */,
519 true /* tosca_live */,
520 true /* expand_call */);
521 }
522 }
523
524 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
525 Address dst, Register val, Register tmp1, Register tmp2) {
526
527 bool in_heap = (decorators & IN_HEAP) != 0;
528 bool in_concurrent_root = (decorators & IN_CONCURRENT_ROOT) != 0;
529 if (in_heap) {
530 write_barrier(masm, dst.base());
531 }
532 if (type == T_OBJECT || type == T_ARRAY) {
533 bool needs_pre_barrier = in_heap || in_concurrent_root;
534 bool needs_post_barrier = val != noreg && in_heap && UseShenandoahMatrix;
535
536 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
537 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
538 // flatten object address if needed
539 // We do it regardless of precise because we need the registers
540 if (dst.index() == noreg && dst.disp() == 0) {
541 if (dst.base() != tmp1) {
542 __ movptr(tmp1, dst.base());
543 }
544 } else {
545 __ lea(tmp1, dst);
546 }
547
548 #ifndef _LP64
549 InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
550 #endif
551
552 NOT_LP64(__ get_thread(rcx));
553 NOT_LP64(imasm->save_bcp());
554
555 if (needs_pre_barrier) {
556 shenandoah_write_barrier_pre(masm /*masm*/,
557 tmp1 /* obj */,
558 tmp2 /* pre_val */,
559 rthread /* thread */,
560 tmp3 /* tmp */,
561 val != noreg /* tosca_live */,
562 false /* expand_call */);
563 }
564 if (val == noreg) {
565 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
566 } else {
567 storeval_barrier(masm, val, tmp3);
568 Register new_val = val;
569 if (needs_post_barrier) {
570 if (UseCompressedOops) {
571 new_val = tmp2;
572 __ movptr(new_val, val);
573 }
574 }
575 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
576
577 if (needs_post_barrier) {
578 shenandoah_write_barrier_post(masm /*masm*/,
579 tmp1 /* store_adr */,
580 new_val /* new_val */,
581 rthread /* thread */,
582 tmp3 /* tmp */,
583 tmp2 /* tmp2 */);
584 }
585 }
586 NOT_LP64(imasm->restore_bcp());
587
588 } else {
589 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
590 }
591 }
592
593 void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, DecoratorSet decorators, Register op1, Register op2) {
594 __ cmpptr(op1, op2);
595 if (ShenandoahAcmpBarrier) {
596 Label done;
597 __ jccb(Assembler::equal, done);
598 read_barrier(masm, op1);
599 read_barrier(masm, op2);
600 __ cmpptr(op1, op2);
601 __ bind(done);
602 }
603 }
604
605 void ShenandoahBarrierSetAssembler::obj_equals_addr(MacroAssembler* masm, DecoratorSet decorators, Register src1, Address src2) {
606 __ cmpptr(src1, src2);
607 if (ShenandoahAcmpBarrier) {
608 Label done;
609 __ jccb(Assembler::equal, done);
610 __ movptr(rscratch2, src2);
611 read_barrier(masm, src1);
612 read_barrier(masm, rscratch2);
613 __ cmpptr(src1, rscratch2);
614 __ bind(done);
615 }
616 }
617
618 void ShenandoahBarrierSetAssembler::resolve_for_read(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
619 bool oop_not_null = (decorators & OOP_NOT_NULL) != 0;
620 if (oop_not_null) {
621 read_barrier_not_null(masm, obj);
622 } else {
623 read_barrier(masm, obj);
624 }
625 }
626
627 void ShenandoahBarrierSetAssembler::resolve_for_write(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
628 write_barrier(masm, obj);
629 }
630
631 // Special Shenandoah CAS implementation that handles false negatives
632 // due to concurrent evacuation.
633 #ifndef _LP64
634 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, DecoratorSet decorators,
635 Register res, Address addr, Register oldval, Register newval,
636 bool exchange, bool encode, Register tmp1, Register tmp2) {
637 // Shenandoah has no 32-bit version for this.
638 Unimplemented();
639 }
640 #else
641 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, DecoratorSet decorators,
642 Register res, Address addr, Register oldval, Register newval,
643 bool exchange, bool encode, Register tmp1, Register tmp2) {
644
645 if (!ShenandoahCASBarrier) {
646 BarrierSetAssembler::cmpxchg_oop(masm, decorators, res, addr, oldval, newval, exchange, encode, tmp1, tmp2);
647 return;
648 }
649
650 assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
651 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
652
653 Label retry, done;
654
655 // Apply storeval barrier to newval.
656 if (encode) {
657 if (newval == c_rarg1 && ShenandoahStoreValEnqueueBarrier) {
658 __ mov(tmp2, newval);
659 storeval_barrier(masm, tmp2, tmp1);
660 } else {
661 storeval_barrier(masm, newval, tmp1);
662 }
663 }
664
665 if (UseCompressedOops) {
666 if (encode) {
667 __ encode_heap_oop(oldval);
668 __ mov(rscratch1, newval);
669 __ encode_heap_oop(rscratch1);
670 newval = rscratch1;
671 }
672 }
673
674 // Remember oldval for retry logic below
675 if (UseCompressedOops) {
676 __ movl(tmp1, oldval);
677 } else {
678 __ movptr(tmp1, oldval);
679 }
680
681 // Step 1. Try to CAS with given arguments. If successful, then we are done,
682 // and can safely return.
683 if (os::is_MP()) __ lock();
684 if (UseCompressedOops) {
685 __ cmpxchgl(newval, addr);
686 } else {
687 __ cmpxchgptr(newval, addr);
688 }
689 __ jcc(Assembler::equal, done, true);
690
691 // Step 2. CAS had failed. This may be a false negative.
692 //
693 // The trouble comes when we compare the to-space pointer with the from-space
694 // pointer to the same object. To resolve this, it will suffice to read both
695 // oldval and the value from memory through the read barriers -- this will give
696 // both to-space pointers. If they mismatch, then it was a legitimate failure.
697 //
698 if (UseCompressedOops) {
699 __ decode_heap_oop(tmp1);
700 }
701 __ resolve_for_read(0, tmp1);
702
703 if (UseCompressedOops) {
704 __ movl(tmp2, oldval);
705 __ decode_heap_oop(tmp2);
706 } else {
707 __ movptr(tmp2, oldval);
708 }
709 __ resolve_for_read(0, tmp2);
710
711 __ cmpptr(tmp1, tmp2);
712 __ jcc(Assembler::notEqual, done, true);
713
714 // Step 3. Try to CAS again with resolved to-space pointers.
715 //
716 // Corner case: it may happen that somebody stored the from-space pointer
717 // to memory while we were preparing for retry. Therefore, we can fail again
718 // on retry, and so need to do this in loop, always re-reading the failure
719 // witness through the read barrier.
720 __ bind(retry);
721 if (os::is_MP()) __ lock();
722 if (UseCompressedOops) {
723 __ cmpxchgl(newval, addr);
724 } else {
725 __ cmpxchgptr(newval, addr);
726 }
727 __ jcc(Assembler::equal, done, true);
728
729 if (UseCompressedOops) {
730 __ movl(tmp2, oldval);
731 __ decode_heap_oop(tmp2);
732 } else {
733 __ movptr(tmp2, oldval);
734 }
735 __ resolve_for_read(0, tmp2);
736
737 __ cmpptr(tmp1, tmp2);
738 __ jcc(Assembler::equal, retry, true);
739
740 // Step 4. If we need a boolean result out of CAS, check the flag again,
741 // and promote the result. Note that we handle the flag from both the CAS
742 // itself and from the retry loop.
743 __ bind(done);
744 if (!exchange) {
745 assert(res != NULL, "need result register");
746 __ setb(Assembler::equal, res);
747 __ movzbl(res, res);
748 }
749 }
750 #endif // LP64
751
752 void ShenandoahBarrierSetAssembler::xchg_oop(MacroAssembler* masm, DecoratorSet decorators,
753 Register obj, Address addr, Register tmp) {
754 storeval_barrier(masm, obj, tmp);
755 BarrierSetAssembler::xchg_oop(masm, decorators, obj, addr, tmp);
756 }
757
758 #ifdef COMPILER1
759
760 #undef __
761 #define __ ce->masm()->
762
763 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
764 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
765 // At this point we know that marking is in progress.
766 // If do_load() is true then we have to emit the
767 // load of the previous value; otherwise it has already
768 // been loaded into _pre_val.
769
770 __ bind(*stub->entry());
771 assert(stub->pre_val()->is_register(), "Precondition.");
772
773 Register pre_val_reg = stub->pre_val()->as_register();
774
775 if (stub->do_load()) {
776 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
777 }
778
779 __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
780 __ jcc(Assembler::equal, *stub->continuation());
781 ce->store_parameter(stub->pre_val()->as_register(), 0);
782 __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
783 __ jmp(*stub->continuation());
784
785 }
786
787 #undef __
788
789 #define __ sasm->
790
791 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
792 __ prologue("shenandoah_pre_barrier", false);
793 // arg0 : previous value of memory
794
795 __ push(rax);
796 __ push(rdx);
797
798 const Register pre_val = rax;
799 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
800 const Register tmp = rdx;
801
802 NOT_LP64(__ get_thread(thread);)
803
804 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
805 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
806
807 Label done;
808 Label runtime;
809
810 // Is SATB still active?
811 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
812 __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
813 __ jcc(Assembler::zero, done);
814
815 // Can we store original value in the thread's buffer?
816
817 __ movptr(tmp, queue_index);
818 __ testptr(tmp, tmp);
819 __ jcc(Assembler::zero, runtime);
820 __ subptr(tmp, wordSize);
821 __ movptr(queue_index, tmp);
822 __ addptr(tmp, buffer);
823
824 // prev_val (rax)
825 __ load_parameter(0, pre_val);
826 __ movptr(Address(tmp, 0), pre_val);
827 __ jmp(done);
828
829 __ bind(runtime);
830
831 __ save_live_registers_no_oop_map(true);
832
833 // load the pre-value
834 __ load_parameter(0, rcx);
835 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
836
837 __ restore_live_registers(true);
838
839 __ bind(done);
840
841 __ pop(rdx);
842 __ pop(rax);
843
844 __ epilogue();
845 }
846
847 #undef __
848
849 #endif // COMPILER1
850
851 address ShenandoahBarrierSetAssembler::shenandoah_wb() {
852 assert(_shenandoah_wb != NULL, "need write barrier stub");
853 return _shenandoah_wb;
854 }
855
856 address ShenandoahBarrierSetAssembler::shenandoah_wb_C() {
857 assert(_shenandoah_wb_C != NULL, "need write barrier stub");
858 return _shenandoah_wb_C;
859 }
860
861 #define __ cgen->assembler()->
862
863 address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen, bool c_abi, bool do_cset_test) {
864 __ align(CodeEntryAlignment);
865 StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
866 address start = __ pc();
867
868 #ifdef _LP64
869 Label not_done;
870
871 // We use RDI, which also serves as argument register for slow call.
872 // RAX always holds the src object ptr, except after the slow call and
873 // the cmpxchg, then it holds the result.
874 // R8 and RCX are used as temporary registers.
875 if (!c_abi) {
876 __ push(rdi);
877 __ push(r8);
878 }
879
880 // Check for object beeing in the collection set.
881 // TODO: Can we use only 1 register here?
882 // The source object arrives here in rax.
883 // live: rax
884 // live: rdi
885 if (!c_abi) {
886 __ mov(rdi, rax);
887 } else {
888 if (rax != c_rarg0) {
889 __ mov(rax, c_rarg0);
890 }
891 }
892 if (do_cset_test) {
893 __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
894 // live: r8
895 __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
896 __ movbool(r8, Address(r8, rdi, Address::times_1));
897 // unlive: rdi
898 __ testbool(r8);
899 // unlive: r8
900 __ jccb(Assembler::notZero, not_done);
901
902 if (!c_abi) {
903 __ pop(r8);
904 __ pop(rdi);
905 }
906 __ ret(0);
907
908 __ bind(not_done);
909 }
910
911 if (!c_abi) {
912 __ push(rcx);
913 }
914
915 if (!c_abi) {
916 __ push(rdx);
917 __ push(rdi);
918 __ push(rsi);
919 __ push(r8);
920 __ push(r9);
921 __ push(r10);
922 __ push(r11);
923 __ push(r12);
924 __ push(r13);
925 __ push(r14);
926 __ push(r15);
927 }
928 __ save_vector_registers();
929 __ movptr(rdi, rax);
930 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
931 __ restore_vector_registers();
932 if (!c_abi) {
933 __ pop(r15);
934 __ pop(r14);
935 __ pop(r13);
936 __ pop(r12);
937 __ pop(r11);
938 __ pop(r10);
939 __ pop(r9);
940 __ pop(r8);
941 __ pop(rsi);
942 __ pop(rdi);
943 __ pop(rdx);
944
945 __ pop(rcx);
946 __ pop(r8);
947 __ pop(rdi);
948 }
949 __ ret(0);
950 #else
951 ShouldNotReachHere();
952 #endif
953 return start;
954 }
955
956 #undef __
957
958 void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
959 if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
960 int stub_code_size = 4096;
961 ResourceMark rm;
962 BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
963 CodeBuffer buf(bb);
964 StubCodeGenerator cgen(&buf);
965 _shenandoah_wb = generate_shenandoah_wb(&cgen, false, true);
966 _shenandoah_wb_C = generate_shenandoah_wb(&cgen, true, !ShenandoahWriteBarrierCsetTestInIR);
967 }
968 }